Add memtable API (CEP-11)

patch by Branimir Lambov; reviewed by Andrés de la Peña and Caleb Rackliffe for CASSANDRA-17034

.build/build-rat.xml

@@ -79,7 +79,7 @@
                  <exclude name="**/tools/cqlstress-lwt-example.yaml"/>
                  <!-- Documentation files -->
                  <exclude NAME="**/doc/modules/**/*"/>
-                 <exclude NAME="**/src/java/**/Paxos.md"/>
+                 <exclude NAME="**/src/java/**/*.md"/>
                  <!-- NOTICE files -->
                  <exclude NAME="**/NOTICE.md"/>
                  <!-- LICENSE files -->

CHANGES.txt

@@ -1,4 +1,5 @@
 4.1
+ * Add a pluggable memtable API (CEP-11 / CASSANDRA-17034)
  * Save sstable id as string in activity table (CASSANDRA-17585)
  * Implement startup check to prevent Cassandra to potentially spread zombie data (CASSANDRA-17180)
  * Allow failing startup on duplicate config keys (CASSANDRA-17379)

NEWS.txt

@@ -56,6 +56,8 @@ using the provided 'sstableupgrade' tool.
 
 New features
 ------------
+    - Added API for alternative memtable implementations. For details, see
+      src/java/org/apache/cassandra/db/memtable/Memtable_API.md
     - Added a new guardrails framework allowing to define soft/hard limits for different user actions, such as limiting
       the number of tables, columns per table or the size of collections. These guardrails are only applied to regular
       user queries, and superusers and internal queries are excluded. Reaching the soft limit raises a client warning,
@@ -88,7 +90,7 @@ New features
         - Whether querying with ALLOW FILTERING is allowed.
     - Add support for the use of pure monotonic functions on the last attribute of the GROUP BY clause.
     - Add floor functions that can be use to group by time range.
-    - Support for native transport rate limiting via native_transport_rate_limiting_enabled and 
+    - Support for native transport rate limiting via native_transport_rate_limiting_enabled and
       native_transport_max_requests_per_second in cassandra.yaml.
     - Support for pre hashing passwords on CQL DCL commands
     - Expose all client options via system_views.clients and nodetool clientstats.

build.xml

@@ -1369,10 +1369,12 @@
 
   <target name="build-jmh" depends="build-test, jar" description="Create JMH uber jar">
       <jar jarfile="${build.test.dir}/deps.jar">
-          <zipgroupfileset dir="${build.dir.lib}/jars">
+          <zipgroupfileset dir="${test.lib}/jars">
               <include name="*jmh*.jar"/>
               <include name="jopt*.jar"/>
               <include name="commons*.jar"/>
+              <include name="junit*.jar"/>
+              <include name="hamcrest*.jar"/>
           </zipgroupfileset>
           <zipgroupfileset dir="${build.lib}" includes="*.jar"/>
       </jar>

conf/commitlog_archiving.properties

@@ -44,5 +44,14 @@ restore_directories=
 # or equal to this timestamp will be applied.
 restore_point_in_time=
 
+# Snapshot commit log position override. This should not be normally necessary, unless the snapshot used a method other
+# than sstables to store data (e.g. persistent memtable was restored from snapshot).
+# Format: segmentId, position
+#
+# Recovery will not replay any commit log data before the specified commit log position. It will NOT exclude the
+# intervals covered by existing sstables from the replay interval, i.e. it will replay data that may already be in
+# sstables.
+snapshot_commitlog_position=
+
 # precision of the timestamp used in the inserts (MILLISECONDS, MICROSECONDS, ...)
 precision=MICROSECONDS

pylib/cqlshlib/test/test_cqlsh_output.py

@@ -658,6 +658,7 @@ class TestCqlshOutput(BaseTestCase):
                 AND comment = ''
                 AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
                 AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+                AND memtable = 'default'
                 AND crc_check_chance = 1.0
                 AND default_time_to_live = 0
                 AND extensions = {}

src/java/org/apache/cassandra/config/Config.java

@@ -21,6 +21,7 @@ import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -174,6 +175,17 @@ public class Config
     public SmallestDataStorageMebibytes memtable_offheap_space;
     public Float memtable_cleanup_threshold = null;
 
+    public static class MemtableOptions
+    {
+        public LinkedHashMap<String, InheritingClass> configurations; // order must be preserved
+
+        public MemtableOptions()
+        {
+        }
+    }
+
+    public MemtableOptions memtable;
+
     // Limit the maximum depth of repair session merkle trees
     @Deprecated
     public volatile Integer repair_session_max_tree_depth = null;

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -3326,6 +3326,13 @@ public class DatabaseDescriptor
         return conf.memtable_cleanup_threshold;
     }
 
+    public static Map<String, InheritingClass> getMemtableConfigurations()
+    {
+        if (conf == null || conf.memtable == null)
+            return null;
+        return conf.memtable.configurations;
+    }
+
     public static int getIndexSummaryResizeIntervalInMinutes()
     {
         return conf.index_summary_resize_interval.toMinutesAsInt();

src/java/org/apache/cassandra/config/InheritingClass.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+public class InheritingClass extends ParameterizedClass
+{
+    public String inherits = null;
+
+    @SuppressWarnings("unused") // for snakeyaml
+    public InheritingClass()
+    {
+    }
+
+    public InheritingClass(String inherits, String class_name, Map<String, String> parameters)
+    {
+        super(class_name, parameters);
+        this.inherits = inherits;
+    }
+
+    @SuppressWarnings("unused")
+    public InheritingClass(Map<String, ?> p)
+    {
+        super(p);
+        this.inherits = p.get("inherits").toString();
+    }
+
+    public ParameterizedClass resolve(Map<String, ParameterizedClass> map)
+    {
+        if (inherits == null)
+            return this;
+        ParameterizedClass parent = map.get(inherits);
+        if (parent == null)
+            throw new ConfigurationException("Configuration definition inherits unknown " + inherits
+                                             + ". A configuration can only extend one defined earlier or \"default\".");
+        Map<String, String> resolvedParameters;
+        if (parameters == null || parameters.isEmpty())
+            resolvedParameters = parent.parameters;
+        else if (parent.parameters == null || parent.parameters.isEmpty())
+            resolvedParameters = this.parameters;
+        else
+        {
+            resolvedParameters = new LinkedHashMap<>(parent.parameters);
+            resolvedParameters.putAll(this.parameters);
+        }
+
+        String resolvedClass = this.class_name == null ? parent.class_name : this.class_name;
+        return new ParameterizedClass(resolvedClass, resolvedParameters);
+    }
+
+    @Override
+    public String toString()
+    {
+        return (inherits != null ? (inherits + "+") : "") +
+               (class_name != null ? class_name : "") +
+               (parameters != null ? parameters.toString() : "");
+    }
+}

src/java/org/apache/cassandra/config/YamlConfigurationLoader.java

@@ -324,6 +324,10 @@ public class YamlConfigurationLoader implements ConfigurationLoader
             TypeDescription seedDesc = new TypeDescription(ParameterizedClass.class);
             seedDesc.putMapPropertyType("parameters", String.class, String.class);
             addTypeDescription(seedDesc);
+
+            TypeDescription memtableDesc = new TypeDescription(Config.MemtableOptions.class);
+            memtableDesc.addPropertyParameters("configurations", String.class, InheritingClass.class);
+            addTypeDescription(memtableDesc);
         }
 
         @Override

src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.cql3.statements.schema;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
@@ -29,6 +30,7 @@ import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.schema.MemtableParams;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.schema.TableParams.Option;
@@ -121,6 +123,9 @@ public final class TableAttributes extends PropertyDefinitions
             builder.compression(CompressionParams.fromMap(getMap(Option.COMPRESSION)));
         }
 
+        if (hasOption(Option.MEMTABLE))
+            builder.memtable(MemtableParams.get(getString(Option.MEMTABLE)));
+
         if (hasOption(Option.DEFAULT_TIME_TO_LIVE))
             builder.defaultTimeToLive(getInt(Option.DEFAULT_TIME_TO_LIVE));
 

src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java

@@ -18,9 +18,14 @@
 
 package org.apache.cassandra.db;
 
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
@@ -46,8 +51,7 @@ public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler
             CommitLogPosition position = null;
             if (makeDurable)
             {
-                Tracing.trace("Appending to commitlog");
-                position = CommitLog.instance.add(mutation);
+                position = addToCommitLog(mutation);
             }
             return new CassandraWriteContext(group, position);
         }
@@ -61,6 +65,41 @@ public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler
         }
     }
 
+    private CommitLogPosition addToCommitLog(Mutation mutation)
+    {
+        // Usually one of these will be true, so first check if that's the case.
+        boolean allSkipCommitlog = true;
+        boolean noneSkipCommitlog = true;
+        for (PartitionUpdate update : mutation.getPartitionUpdates())
+        {
+            if (update.metadata().params.memtable.factory().writesShouldSkipCommitLog())
+                noneSkipCommitlog = false;
+            else
+                allSkipCommitlog = false;
+        }
+
+        if (!noneSkipCommitlog)
+        {
+            if (allSkipCommitlog)
+                return null;
+            else
+            {
+                Set<TableId> ids = new HashSet<>();
+                for (PartitionUpdate update : mutation.getPartitionUpdates())
+                {
+                    if (update.metadata().params.memtable.factory().writesShouldSkipCommitLog())
+                        ids.add(update.metadata().id);
+                }
+                mutation = mutation.without(ids);
+            }
+        }
+        // Note: It may be a good idea to precalculate none/all for the set of all tables in the keyspace,
+        // or memoize the mutation.getTableIds()->ids map (needs invalidation on schema version change).
+
+        Tracing.trace("Appending to commitlog");
+        return CommitLog.instance.add(mutation);
+    }
+
     @SuppressWarnings("resource") // group is closed when CassandraWriteContext is closed
     private WriteContext createEmptyContext()
     {

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -28,9 +28,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -42,6 +44,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -61,6 +64,7 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -75,7 +79,6 @@ import org.apache.cassandra.cache.RowCacheKey;
 import org.apache.cassandra.cache.RowCacheSentinel;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.FutureTask;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.DurationSpec;
 import org.apache.cassandra.db.commitlog.CommitLog;
@@ -87,6 +90,9 @@ import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.compaction.Verifier;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.memtable.Flushing;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.ShardBoundaries;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
@@ -102,6 +108,7 @@ import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.StartupException;
@@ -112,6 +119,7 @@ import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableId;
 import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
@@ -159,14 +167,11 @@ import org.apache.cassandra.utils.MBeanWrapper;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.WrappedRunnable;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.concurrent.Promise;
 import org.apache.cassandra.utils.concurrent.Refs;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
-import org.apache.cassandra.utils.memory.MemtableAllocator;
 
 import static com.google.common.base.Throwables.propagate;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -180,7 +185,7 @@ import static org.apache.cassandra.utils.Throwables.maybeFail;
 import static org.apache.cassandra.utils.Throwables.merge;
 import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
 
-public class ColumnFamilyStore implements ColumnFamilyStoreMBean
+public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 {
     private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class);
 
@@ -209,6 +214,35 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                                                                                                  DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations(),
                                                                                                  DatabaseDescriptor.useSpecificLocationForLocalSystemData());
 
+    /**
+     * Reason for initiating a memtable flush.
+     */
+    public enum FlushReason
+    {
+        COMMITLOG_DIRTY,
+        MEMTABLE_LIMIT,
+        MEMTABLE_PERIOD_EXPIRED,
+        INDEX_BUILD_STARTED,
+        INDEX_BUILD_COMPLETED,
+        INDEX_REMOVED,
+        INDEX_TABLE_FLUSH,
+        VIEW_BUILD_STARTED,
+        INTERNALLY_FORCED,  // explicitly requested flush, necessary for the operation of an internal table
+        USER_FORCED, // flush explicitly requested by the user (e.g. nodetool flush)
+        STARTUP,
+        DRAIN,
+        SNAPSHOT,
+        TRUNCATE,
+        DROP,
+        STREAMING,
+        STREAMS_RECEIVED,
+        VALIDATION,
+        ANTICOMPACTION,
+        SCHEMA_CHANGE,
+        OWNED_RANGES_CHANGE,
+        UNIT_TESTS // explicitly requested flush needed for a test
+    }
+
     private static final String[] COUNTER_NAMES = new String[]{"table", "count", "error", "value"};
     private static final String[] COUNTER_DESCS = new String[]
     { "keyspace.tablename",
@@ -243,6 +277,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     private final String oldMBeanName;
     private volatile boolean valid = true;
 
+    private volatile Memtable.Factory memtableFactory;
+
     /**
      * Memtables and SSTables on disk for this column family.
      *
@@ -287,6 +323,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
     @VisibleForTesting
     final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager();
+    private volatile ShardBoundaries cachedShardBoundaries = null;
 
     private volatile boolean neverPurgeTombstones = false;
 
@@ -344,48 +381,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
         compactionStrategyManager.maybeReload(metadata());
 
-        scheduleFlush();
-
         indexManager.reload();
 
-        // If the CF comparator has changed, we need to change the memtable,
-        // because the old one still aliases the previous comparator.
-        if (data.getView().getCurrentMemtable().initialComparator != metadata().comparator)
-            switchMemtable();
-    }
-
-    void scheduleFlush()
-    {
-        int period = metadata().params.memtableFlushPeriodInMs;
-        if (period > 0)
-        {
-            logger.trace("scheduling flush in {} ms", period);
-            WrappedRunnable runnable = new WrappedRunnable()
-            {
-                protected void runMayThrow()
-                {
-                    synchronized (data)
-                    {
-                        Memtable current = data.getView().getCurrentMemtable();
-                        // if we're not expired, we've been hit by a scheduled flush for an already flushed memtable, so ignore
-                        if (current.isExpired())
-                        {
-                            if (current.isClean())
-                            {
-                                // if we're still clean, instead of swapping just reschedule a flush for later
-                                scheduleFlush();
-                            }
-                            else
-                            {
-                                // we'll be rescheduled by the constructor of the Memtable.
-                                forceFlush();
-                            }
-                        }
-                    }
-                }
-            };
-            ScheduledExecutors.scheduledTasks.scheduleSelfRecurring(runnable, period, TimeUnit.MILLISECONDS);
-        }
+        memtableFactory = metadata().params.memtable.factory();
+        switchMemtableOrNotify(FlushReason.SCHEMA_CHANGE, Memtable::metadataUpdated);
     }
 
     public static Runnable getBackgroundCompactionTaskSubmitter()
@@ -481,14 +480,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         this.sstableIdGenerator = sstableIdGenerator;
         sampleReadLatencyNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) / 2;
         additionalWriteLatencyNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) / 2;
+        memtableFactory = metadata.get().params.memtable.factory();
 
         logger.info("Initializing {}.{}", keyspace.getName(), name);
 
-        // Create Memtable only on online
+        // Create Memtable and its metrics object only on online
         Memtable initialMemtable = null;
+        TableMetrics.ReleasableMetric memtableMetrics = null;
         if (DatabaseDescriptor.isDaemonInitialized())
-            initialMemtable = new Memtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()), this);
-        data = new Tracker(initialMemtable, loadSSTables);
+        {
+            initialMemtable = createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()));
+            memtableMetrics = memtableFactory.createMemtableMetrics(metadata);
+        }
+        data = new Tracker(this, initialMemtable, loadSSTables);
 
         // Note that this needs to happen before we load the first sstables, or the global sstable tracker will not
         // be notified on the initial loading.
@@ -519,7 +523,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             indexManager.addIndex(info, true);
         }
 
-        metric = new TableMetrics(this);
+        metric = new TableMetrics(this, memtableMetrics);
 
         if (data.loadsstables)
         {
@@ -615,6 +619,26 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         return dataPaths;
     }
 
+    public boolean writesShouldSkipCommitLog()
+    {
+        return memtableFactory.writesShouldSkipCommitLog();
+    }
+
+    public boolean memtableWritesAreDurable()
+    {
+        return memtableFactory.writesAreDurable();
+    }
+
+    public boolean streamToMemtable()
+    {
+        return memtableFactory.streamToMemtable();
+    }
+
+    public boolean streamFromMemtable()
+    {
+        return memtableFactory.streamFromMemtable();
+    }
+
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
         MetadataCollector collector = new MetadataCollector(metadata().comparator).sstableLevel(sstableLevel);
@@ -924,17 +948,30 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         return newDescriptor;
     }
 
+    /**
+     * Checks with the memtable if it should be switched for the given reason, and if not, calls the specified
+     * notification method.
+     */
+    private void switchMemtableOrNotify(FlushReason reason, Consumer<Memtable> elseNotify)
+    {
+        Memtable currentMemtable = data.getView().getCurrentMemtable();
+        if (currentMemtable.shouldSwitch(reason))
+            switchMemtableIfCurrent(currentMemtable, reason);
+        else
+            elseNotify.accept(currentMemtable);
+    }
+
     /**
      * Switches the memtable iff the live memtable is the one provided
      *
      * @param memtable
      */
-    public Future<CommitLogPosition> switchMemtableIfCurrent(Memtable memtable)
+    public Future<CommitLogPosition> switchMemtableIfCurrent(Memtable memtable, FlushReason reason)
     {
         synchronized (data)
         {
             if (data.getView().getCurrentMemtable() == memtable)
-                return switchMemtable();
+                return switchMemtable(reason);
         }
         logger.debug("Memtable is no longer current, returning future that completes when current flushing operation completes");
         return waitForFlushes();
@@ -947,11 +984,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
      * not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL
      * marked clean up to the position owned by the Memtable.
      */
-    public Future<CommitLogPosition> switchMemtable()
+    @VisibleForTesting
+    public Future<CommitLogPosition> switchMemtable(FlushReason reason)
     {
         synchronized (data)
         {
-            logFlush();
+            logFlush(reason);
             Flush flush = new Flush(false);
             flushExecutor.execute(flush);
             postFlushExecutor.execute(flush.postFlushTask);
@@ -960,33 +998,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
     // print out size of all memtables we're enqueuing
-    private void logFlush()
+    private void logFlush(FlushReason reason)
     {
         // reclaiming includes that which we are GC-ing;
-        float onHeapRatio = 0, offHeapRatio = 0;
-        long onHeapTotal = 0, offHeapTotal = 0;
-        Memtable memtable = getTracker().getView().getCurrentMemtable();
-        onHeapRatio +=  memtable.getAllocator().onHeap().ownershipRatio();
-        offHeapRatio += memtable.getAllocator().offHeap().ownershipRatio();
-        onHeapTotal += memtable.getAllocator().onHeap().owns();
-        offHeapTotal += memtable.getAllocator().offHeap().owns();
+        Memtable.MemoryUsage usage = Memtable.newMemoryUsage();
+        getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
 
         for (ColumnFamilyStore indexCfs : indexManager.getAllIndexColumnFamilyStores())
-        {
-            MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator();
-            onHeapRatio += allocator.onHeap().ownershipRatio();
-            offHeapRatio += allocator.offHeap().ownershipRatio();
-            onHeapTotal += allocator.onHeap().owns();
-            offHeapTotal += allocator.offHeap().owns();
-        }
+            indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
 
-        logger.info("Enqueuing flush of {}: {}",
-                     name,
-                     String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap",
-                                   FBUtilities.prettyPrintMemory(onHeapTotal),
-                                   onHeapRatio * 100,
-                                   FBUtilities.prettyPrintMemory(offHeapTotal),
-                                   offHeapRatio * 100));
+        logger.info("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", keyspace.getName(), name, reason, usage);
     }
 
 
@@ -996,14 +1017,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
      * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
      *         to sstables for this table once the future completes
      */
-    public Future<CommitLogPosition> forceFlush()
+    public Future<CommitLogPosition> forceFlush(FlushReason reason)
     {
         synchronized (data)
         {
             Memtable current = data.getView().getCurrentMemtable();
             for (ColumnFamilyStore cfs : concatWithIndexes())
                 if (!cfs.data.getView().getCurrentMemtable().isClean())
-                    return switchMemtableIfCurrent(current);
+                    return flushMemtable(current, reason);
             return waitForFlushes();
         }
     }
@@ -1021,10 +1042,18 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         // and this does not vary between a table and its table-backed indexes
         Memtable current = data.getView().getCurrentMemtable();
         if (current.mayContainDataBefore(flushIfDirtyBefore))
-            return switchMemtableIfCurrent(current);
+            return flushMemtable(current, FlushReason.COMMITLOG_DIRTY);
         return waitForFlushes();
     }
 
+    private Future<CommitLogPosition> flushMemtable(Memtable current, FlushReason reason)
+    {
+        if (current.shouldSwitch(reason))
+            return switchMemtableIfCurrent(current, reason);
+        else
+            return waitForFlushes();
+    }
+
     /**
      * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
      *         to sstables for this table once the future completes
@@ -1034,15 +1063,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         // we grab the current memtable; once any preceding memtables have flushed, we know its
         // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable)
         final Memtable current = data.getView().getCurrentMemtable();
-        return postFlushExecutor.submit(() -> {
-            logger.debug("forceFlush requested but everything is clean in {}", name);
-            return current.getCommitLogLowerBound();
-        });
+        return postFlushExecutor.submit(current::getCommitLogLowerBound);
     }
 
-    public CommitLogPosition forceBlockingFlush()
+    public CommitLogPosition forceBlockingFlush(FlushReason reason)
     {
-        return FBUtilities.waitOnFuture(forceFlush());
+        return FBUtilities.waitOnFuture(forceFlush(reason));
     }
 
     /**
@@ -1052,12 +1078,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     private final class PostFlush implements Callable<CommitLogPosition>
     {
         final CountDownLatch latch = newCountDownLatch(1);
-        final List<Memtable> memtables;
+        final Memtable mainMemtable;
         volatile Throwable flushFailure = null;
 
-        private PostFlush(List<Memtable> memtables)
+        private PostFlush(Memtable mainMemtable)
         {
-            this.memtables = memtables;
+            this.mainMemtable = mainMemtable;
         }
 
         public CommitLogPosition call()
@@ -1075,11 +1101,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             CommitLogPosition commitLogUpperBound = NONE;
             // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
-            if (flushFailure == null && !memtables.isEmpty())
+            if (flushFailure == null && mainMemtable != null)
             {
-                Memtable memtable = memtables.get(0);
-                commitLogUpperBound = memtable.getCommitLogUpperBound();
-                CommitLog.instance.discardCompletedSegments(metadata.id, memtable.getCommitLogLowerBound(), commitLogUpperBound);
+                commitLogUpperBound = mainMemtable.getCommitLogUpperBound();
+                CommitLog.instance.discardCompletedSegments(metadata.id, mainMemtable.getCommitLogLowerBound(), commitLogUpperBound);
             }
 
             metric.pendingFlushes.dec();
@@ -1102,7 +1127,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     private final class Flush implements Runnable
     {
         final OpOrder.Barrier writeBarrier;
-        final List<Memtable> memtables = new ArrayList<>();
+        final Map<ColumnFamilyStore, Memtable> memtables;
         final FutureTask<CommitLogPosition> postFlushTask;
         final PostFlush postFlush;
         final boolean truncate;
@@ -1126,6 +1151,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
              */
             writeBarrier = Keyspace.writeOrder.newBarrier();
 
+            memtables = new LinkedHashMap<>();
+
             // submit flushes for the memtable for any indexed sub-cfses, and our own
             AtomicReference<CommitLogPosition> commitLogUpperBound = new AtomicReference<>();
             for (ColumnFamilyStore cfs : concatWithIndexes())
@@ -1133,10 +1160,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 // switch all memtables, regardless of their dirty status, setting the barrier
                 // so that we can reach a coordinated decision about cleanliness once they
                 // are no longer possible to be modified
-                Memtable newMemtable = new Memtable(commitLogUpperBound, cfs);
+                Memtable newMemtable = cfs.createMemtable(commitLogUpperBound);
                 Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable);
-                oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound);
-                memtables.add(oldMemtable);
+                oldMemtable.switchOut(writeBarrier, commitLogUpperBound);
+                memtables.put(cfs, oldMemtable);
             }
 
             // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log
@@ -1147,7 +1174,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier
             // commit log segment position have also completed, i.e. the memtables are done and ready to flush
             writeBarrier.issue();
-            postFlush = new PostFlush(memtables);
+            postFlush = new PostFlush(Iterables.get(memtables.values(), 0, null));
             postFlushTask = new FutureTask<>(postFlush);
         }
 
@@ -1167,17 +1194,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 logger.trace("Flush task for task {}@{} waited {} ms at the barrier", hashCode(), name, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start));
 
             // mark all memtables as flushing, removing them from the live memtable list
-            for (Memtable memtable : memtables)
-                memtable.cfs.data.markFlushing(memtable);
+            for (Map.Entry<ColumnFamilyStore, Memtable> entry : memtables.entrySet())
+                entry.getKey().data.markFlushing(entry.getValue());
 
             metric.memtableSwitchCount.inc();
 
             try
             {
+                boolean first = true;
                 // Flush "data" memtable with non-cf 2i first;
-                flushMemtable(memtables.get(0), true);
-                for (int i = 1; i < memtables.size(); i++)
-                    flushMemtable(memtables.get(i), false);
+                for (Map.Entry<ColumnFamilyStore, Memtable> entry : memtables.entrySet())
+                {
+                    flushMemtable(entry.getKey(), entry.getValue(), first);
+                    first = false;
+                }
             }
             catch (Throwable t)
             {
@@ -1195,14 +1225,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 logger.trace("Flush task task {}@{} finished", hashCode(), name);
         }
 
-        public Collection<SSTableReader> flushMemtable(Memtable memtable, boolean flushNonCf2i)
+        public Collection<SSTableReader> flushMemtable(ColumnFamilyStore cfs, Memtable memtable, boolean flushNonCf2i)
         {
             if (logger.isTraceEnabled())
                 logger.trace("Flush task task {}@{} flushing memtable {}", hashCode(), name, memtable);
 
             if (memtable.isClean() || truncate)
             {
-                memtable.cfs.replaceFlushed(memtable, Collections.emptyList());
+                cfs.replaceFlushed(memtable, Collections.emptyList());
                 reclaim(memtable);
                 return Collections.emptyList();
             }
@@ -1214,13 +1244,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             List<SSTableReader> sstables = new ArrayList<>();
             try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
             {
-                List<Memtable.FlushRunnable> flushRunnables = null;
+                List<Flushing.FlushRunnable> flushRunnables = null;
                 List<SSTableMultiWriter> flushResults = null;
 
                 try
                 {
                     // flush the memtable
-                    flushRunnables = memtable.flushRunnables(txn);
+                    flushRunnables = Flushing.flushRunnables(cfs, memtable, txn);
                     ExecutorPlus[] executors = perDiskflushExecutors.getExecutorsFor(keyspace.getName(), name);
 
                     for (int i = 0; i < flushRunnables.size(); i++)
@@ -1239,7 +1269,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 }
                 catch (Throwable t)
                 {
-                    t = memtable.abortRunnables(flushRunnables, t);
+                    t = Flushing.abortRunnables(flushRunnables, t);
                     t = txn.abort(t);
                     throw Throwables.propagate(t);
                 }
@@ -1294,9 +1324,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                     }
                 }
             }
-            memtable.cfs.replaceFlushed(memtable, sstables);
+            cfs.replaceFlushed(memtable, sstables);
             reclaim(memtable);
-            memtable.cfs.compactionStrategyManager.compactionLogger.flush(sstables);
+            cfs.compactionStrategyManager.compactionLogger.flush(sstables);
             logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
                          sstables,
                          sstables.size(),
@@ -1316,7 +1346,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 public void runMayThrow()
                 {
                     readBarrier.await();
-                    memtable.setDiscarded();
+                    memtable.discard();
                 }
             }, reclaimExecutor);
         }
@@ -1328,6 +1358,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
+    public Memtable createMemtable(AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        return memtableFactory.create(commitLogUpperBound, metadata, this);
+    }
+
     // atomically set the upper bound for the commit log
     private static void setCommitLogUpperBound(AtomicReference<CommitLogPosition> commitLogUpperBound)
     {
@@ -1345,86 +1380,29 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
-    /**
-     * Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
-     * queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
-     */
-    public static Future<Boolean> flushLargestMemtable()
+    @Override
+    public Future<CommitLogPosition> signalFlushRequired(Memtable memtable, FlushReason reason)
     {
-        float largestRatio = 0f;
-        Memtable largest = null;
-        float liveOnHeap = 0, liveOffHeap = 0;
-        for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
-        {
-            // we take a reference to the current main memtable for the CF prior to snapping its ownership ratios
-            // to ensure we have some ordering guarantee for performing the switchMemtableIf(), i.e. we will only
-            // swap if the memtables we are measuring here haven't already been swapped by the time we try to swap them
-            Memtable current = cfs.getTracker().getView().getCurrentMemtable();
-
-            // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF,
-            // both on- and off-heap, and select the largest of the two ratios to weight this CF
-            float onHeap = 0f, offHeap = 0f;
-            onHeap += current.getAllocator().onHeap().ownershipRatio();
-            offHeap += current.getAllocator().offHeap().ownershipRatio();
-
-            for (ColumnFamilyStore indexCfs : cfs.indexManager.getAllIndexColumnFamilyStores())
-            {
-                MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator();
-                onHeap += allocator.onHeap().ownershipRatio();
-                offHeap += allocator.offHeap().ownershipRatio();
-            }
-
-            float ratio = Math.max(onHeap, offHeap);
-            if (ratio > largestRatio)
-            {
-                largest = current;
-                largestRatio = ratio;
-            }
-
-            liveOnHeap += onHeap;
-            liveOffHeap += offHeap;
-        }
-
-        Promise<Boolean> returnFuture = new AsyncPromise<>();
-
-        if (largest != null)
-        {
-            float usedOnHeap = Memtable.MEMORY_POOL.onHeap.usedRatio();
-            float usedOffHeap = Memtable.MEMORY_POOL.offHeap.usedRatio();
-            float flushingOnHeap = Memtable.MEMORY_POOL.onHeap.reclaimingRatio();
-            float flushingOffHeap = Memtable.MEMORY_POOL.offHeap.reclaimingRatio();
-            float thisOnHeap = largest.getAllocator().onHeap().ownershipRatio();
-            float thisOffHeap = largest.getAllocator().offHeap().ownershipRatio();
-            logger.info("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}",
-                         largest.cfs, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap),
-                         ratio(flushingOnHeap, flushingOffHeap), ratio(thisOnHeap, thisOffHeap));
-
-            Future<CommitLogPosition> flushFuture = largest.cfs.switchMemtableIfCurrent(largest);
-            flushFuture.addListener(() -> {
-                try
-                {
-                    flushFuture.get();
-                    returnFuture.trySuccess(true);
-                }
-                catch (Throwable t)
-                {
-                    returnFuture.tryFailure(t);
-                }
-            });
-        }
-        else
-        {
-            logger.debug("Flushing of largest memtable, not done, no memtable found");
-
-            returnFuture.trySuccess(false);
-        }
-
-        return returnFuture;
+        return switchMemtableIfCurrent(memtable, reason);
     }
 
-    private static String ratio(float onHeap, float offHeap)
+    @Override
+    public Memtable getCurrentMemtable()
     {
-        return String.format("%.2f/%.2f", onHeap, offHeap);
+        return data.getView().getCurrentMemtable();
+    }
+
+    public static Iterable<Memtable> activeMemtables()
+    {
+        return Iterables.transform(ColumnFamilyStore.all(),
+                                   cfs -> cfs.getTracker().getView().getCurrentMemtable());
+    }
+
+    @Override
+    public Iterable<Memtable> getIndexMemtables()
+    {
+        return Iterables.transform(indexManager.getAllIndexColumnFamilyStores(),
+                                   cfs -> cfs.getTracker().getView().getCurrentMemtable());
     }
 
     /**
@@ -1465,6 +1443,44 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
+    @Override
+    public ShardBoundaries localRangeSplits(int shardCount)
+    {
+        if (shardCount == 1 || !getPartitioner().splitter().isPresent() || SchemaConstants.isLocalSystemKeyspace(keyspace.getName()))
+            return ShardBoundaries.NONE;
+
+        ShardBoundaries shardBoundaries = cachedShardBoundaries;
+        if (shardBoundaries == null ||
+            shardBoundaries.shardCount() != shardCount ||
+            shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
+        {
+            DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
+            Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
+            List<Splitter.WeightedRange> weightedRanges;
+            if (localRanges.isEmpty())
+                weightedRanges = ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
+            else
+            {
+                weightedRanges = new ArrayList<>(localRanges.size());
+                for (Range<Token> r : localRanges)
+                {
+                    // WeightedRange supports only unwrapped ranges as it relies
+                    // on right - left == num tokens equality
+                    for (Range<Token> u: r.unwrap())
+                        weightedRanges.add(new Splitter.WeightedRange(1.0, u));
+                }
+                weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+            }
+
+            List<Token> boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false);
+            shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1),
+                                                  versionedLocalRanges.ringVersion);
+            cachedShardBoundaries = shardBoundaries;
+            logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), boundaries);
+        }
+        return shardBoundaries;
+    }
+
     /**
      * @param sstables
      * @return sstables whose key range overlaps with that of the given sstables, not including itself.
@@ -1634,7 +1650,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         {
             Instant creationTime = now();
             String snapshotName = "pre-scrub-" + creationTime.toEpochMilli();
-            snapshotWithoutFlush(snapshotName, creationTime);
+            snapshotWithoutMemtable(snapshotName, creationTime);
         }
 
         try
@@ -1973,20 +1989,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         return metadata().comparator;
     }
 
-    public TableSnapshot snapshotWithoutFlush(String snapshotName)
+    public TableSnapshot snapshotWithoutMemtable(String snapshotName)
     {
-        return snapshotWithoutFlush(snapshotName, now());
+        return snapshotWithoutMemtable(snapshotName, now());
     }
 
-    public TableSnapshot snapshotWithoutFlush(String snapshotName, Instant creationTime)
+    public TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime)
     {
-        return snapshotWithoutFlush(snapshotName, null, false, null, null, creationTime);
+        return snapshotWithoutMemtable(snapshotName, null, false, null, null, creationTime);
     }
 
     /**
      * @param ephemeral If this flag is set to true, the snapshot will be cleaned during next startup
      */
-    public TableSnapshot snapshotWithoutFlush(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
+    public TableSnapshot snapshotWithoutMemtable(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
     {
         if (ephemeral && ttl != null)
         {
@@ -2175,40 +2191,47 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
      * Take a snap shot of this columnfamily store.
      *
      * @param snapshotName the name of the associated with the snapshot
-     * @param skipFlush Skip blocking flush of memtable
+     * @param skipMemtable Skip flushing the memtable
      * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed
      * @param rateLimiter Rate limiter for hardlinks-per-second
      * @param creationTime time when this snapshot was taken
      */
-    public TableSnapshot snapshot(String snapshotName, boolean skipFlush, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
+    public TableSnapshot snapshot(String snapshotName, boolean skipMemtable, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
     {
-        return snapshot(snapshotName, null, false, skipFlush, ttl, rateLimiter, creationTime);
+        return snapshot(snapshotName, null, false, skipMemtable, ttl, rateLimiter, creationTime);
     }
 
 
     /**
      * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup
-     * @param skipFlush Skip blocking flush of memtable
+     * @param skipMemtable Skip flushing the memtable
      */
-    public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipFlush)
+    public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipMemtable)
     {
-        return snapshot(snapshotName, predicate, ephemeral, skipFlush, null, null, now());
+        return snapshot(snapshotName, predicate, ephemeral, skipMemtable, null, null, now());
     }
 
     /**
      * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup
-     * @param skipFlush Skip blocking flush of memtable
+     * @param skipMemtable Skip flushing the memtable
      * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed
      * @param rateLimiter Rate limiter for hardlinks-per-second
      * @param creationTime time when this snapshot was taken
      */
-    public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipFlush, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
+    public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipMemtable, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime)
     {
-        if (!skipFlush)
+        if (!skipMemtable)
         {
-            forceBlockingFlush();
+            Memtable current = getTracker().getView().getCurrentMemtable();
+            if (!current.isClean())
+            {
+                if (current.shouldSwitch(FlushReason.SNAPSHOT))
+                    FBUtilities.waitOnFuture(switchMemtableIfCurrent(current, FlushReason.SNAPSHOT));
+                else
+                    current.performSnapshot(snapshotName);
+            }
         }
-        return snapshotWithoutFlush(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime);
+        return snapshotWithoutMemtable(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime);
     }
 
     public boolean snapshotExists(String snapshotName)
@@ -2414,6 +2437,108 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
+    public void writeAndAddMemtableRanges(TimeUUID repairSessionID,
+                                          Supplier<Collection<Range<PartitionPosition>>> rangesSupplier,
+                                          Refs<SSTableReader> placeIntoRefs)
+    {
+        @SuppressWarnings("resource") // closed by finish or on exception
+        SSTableMultiWriter memtableContent = writeMemtableRanges(rangesSupplier, repairSessionID);
+        if (memtableContent != null)
+        {
+            try
+            {
+                Collection<SSTableReader> sstables = memtableContent.finish(true);
+                try (Refs sstableReferences = Refs.ref(sstables))
+                {
+                    // This moves all references to placeIntoRefs, clearing sstableReferences
+                    placeIntoRefs.addAll(sstableReferences);
+                }
+
+                // Release the reference any written sstables start with.
+                for (SSTableReader rdr : sstables)
+                {
+                    rdr.selfRef().release();
+                    logger.info("Memtable ranges (keys {} size {}) written in {}",
+                                rdr.estimatedKeys(),
+                                rdr.getDataChannel().size(),
+                                rdr);
+                }
+            }
+            catch (Throwable t)
+            {
+                memtableContent.close();
+                Throwables.propagate(t);
+            }
+        }
+    }
+
+    private SSTableMultiWriter writeMemtableRanges(Supplier<Collection<Range<PartitionPosition>>> rangesSupplier,
+                                                   TimeUUID repairSessionID)
+    {
+        if (!streamFromMemtable())
+            return null;
+
+        Collection<Range<PartitionPosition>> ranges = rangesSupplier.get();
+        Memtable current = getTracker().getView().getCurrentMemtable();
+        if (current.isClean())
+            return null;
+
+        List<Memtable.FlushablePartitionSet<?>> dataSets = new ArrayList<>(ranges.size());
+        long keys = 0;
+        for (Range<PartitionPosition> range : ranges)
+        {
+            Memtable.FlushablePartitionSet<?> dataSet = current.getFlushSet(range.left, range.right);
+            dataSets.add(dataSet);
+            keys += dataSet.partitionCount();
+        }
+        if (keys == 0)
+            return null;
+
+        // TODO: Can we write directly to stream, skipping disk?
+        Memtable.FlushablePartitionSet<?> firstDataSet = dataSets.get(0);
+        SSTableMultiWriter writer = createSSTableMultiWriter(newSSTableDescriptor(directories.getDirectoryForNewSSTables()),
+                                                             keys,
+                                                             0,
+                                                             repairSessionID,
+                                                             false,
+                                                             0,
+                                                             new SerializationHeader(true,
+                                                                                     firstDataSet.metadata(),
+                                                                                     firstDataSet.columns(),
+                                                                                     firstDataSet.encodingStats()),
+                                                             DO_NOT_TRACK);
+        try
+        {
+            for (Memtable.FlushablePartitionSet<?> dataSet : dataSets)
+                new Flushing.FlushRunnable(dataSet, writer, metric, false).call();  // executes on this thread
+
+            return writer;
+        }
+        catch (Error | RuntimeException t)
+        {
+            writer.abort(t);
+            throw t;
+        }
+    }
+
+    private static final LifecycleNewTracker DO_NOT_TRACK = new LifecycleNewTracker()
+    {
+        public void trackNew(SSTable table)
+        {
+            // not tracking
+        }
+
+        public void untrackNew(SSTable table)
+        {
+            // not tracking
+        }
+
+        public OperationType opType()
+        {
+            return OperationType.FLUSH;
+        }
+    };
+
     /**
      * For testing.  No effort is made to clear historical or even the current memtables, nor for
      * thread safety.  All we do is wipe the sstable containers clean, while leaving the actual
@@ -2425,7 +2550,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         for (final ColumnFamilyStore cfs : concatWithIndexes())
         {
             cfs.runWithCompactionsDisabled((Callable<Void>) () -> {
-                cfs.data.reset(new Memtable(new AtomicReference<>(CommitLogPosition.NONE), cfs));
+                cfs.data.reset(memtableFactory.create(new AtomicReference<>(CommitLogPosition.NONE), cfs.metadata, cfs));
                 return null;
             }, true, false);
         }
@@ -2466,23 +2591,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final long truncatedAt;
         final CommitLogPosition replayAfter;
 
-        if (!noSnapshot && (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot()))
+        if (!noSnapshot &&
+               ((keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable())  // need to clear dirty regions
+               || DatabaseDescriptor.isAutoSnapshot())) // need sstable for snapshot
         {
-            replayAfter = forceBlockingFlush();
-            viewManager.forceBlockingFlush();
+            replayAfter = forceBlockingFlush(FlushReason.TRUNCATE);
+            viewManager.forceBlockingFlush(FlushReason.TRUNCATE);
         }
         else
         {
             // just nuke the memtable data w/o writing to disk first
+            // note: this does not wait for the switch to complete, but because the post-flush processing is serial,
+            // the call below does.
             viewManager.dumpMemtables();
-            try
-            {
-                replayAfter = dumpMemtable().get();
-            }
-            catch (Exception e)
-            {
-                throw new RuntimeException(e);
-            }
+            replayAfter = FBUtilities.waitOnFuture(dumpMemtable());
         }
 
         long now = currentTimeMillis();
@@ -2527,7 +2649,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     }
 
     /**
-     * Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable.
+     * Drops current memtable without flushing to disk. This should only be called when truncating a column family
+     * that cannot have dirty intervals in the commit log (i.e. one which is not durable, or where the memtable itself
+     * performs durable writes).
      */
     public Future<CommitLogPosition> dumpMemtable()
     {
@@ -2540,6 +2664,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
+    public void unloadCf()
+    {
+        if (keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable())  // need to clear dirty regions
+            forceBlockingFlush(ColumnFamilyStore.FlushReason.DROP);
+        else
+            FBUtilities.waitOnFuture(dumpMemtable());
+    }
+
     public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
     {
         return runWithCompactionsDisabled(callable, (sstable) -> true, interruptValidation, interruptViews, true);
@@ -3026,9 +3158,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         return diskBoundaryManager.getDiskBoundaries(this);
     }
 
-    public void invalidateDiskBoundaries()
+    public void invalidateLocalRanges()
     {
         diskBoundaryManager.invalidate();
+
+        switchMemtableOrNotify(FlushReason.OWNED_RANGES_CHANGE, Memtable::localRangesUpdated);
     }
 
     @Override

src/java/org/apache/cassandra/db/DiskBoundaryManager.java

@@ -67,7 +67,19 @@ public class DiskBoundaryManager
            diskBoundaries.invalidate();
     }
 
-    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
+    static class VersionedRangesAtEndpoint
+    {
+        public final RangesAtEndpoint rangesAtEndpoint;
+        public final long ringVersion;
+
+        VersionedRangesAtEndpoint(RangesAtEndpoint rangesAtEndpoint, long ringVersion)
+        {
+            this.rangesAtEndpoint = rangesAtEndpoint;
+            this.ringVersion = ringVersion;
+        }
+    }
+
+    public static VersionedRangesAtEndpoint getVersionedLocalRanges(ColumnFamilyStore cfs)
     {
         RangesAtEndpoint localRanges;
 
@@ -77,23 +89,20 @@ public class DiskBoundaryManager
         {
             tmd = StorageService.instance.getTokenMetadata();
             ringVersion = tmd.getRingVersion();
-            if (StorageService.instance.isBootstrapMode()
-                && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
-            {
-                PendingRangeCalculatorService.instance.blockUntilFinished();
-                localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
-            }
-            else
-            {
-                // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
-                // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
-                // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
-                localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
-            }
+            localRanges = getLocalRanges(cfs, tmd);
             logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion);
         }
         while (ringVersion != tmd.getRingVersion()); // if ringVersion is different here it means that
-                                                     // it might have changed before we calculated localRanges - recalculate
+        // it might have changed before we calculated localRanges - recalculate
+
+        return new VersionedRangesAtEndpoint(localRanges, ringVersion);
+    }
+
+    private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs)
+    {
+        VersionedRangesAtEndpoint rangesAtEndpoint = getVersionedLocalRanges(cfs);
+        RangesAtEndpoint localRanges = rangesAtEndpoint.rangesAtEndpoint;
+        long ringVersion = rangesAtEndpoint.ringVersion;
 
         int directoriesVersion;
         Directories.DataDirectory[] dirs;
@@ -112,6 +121,25 @@ public class DiskBoundaryManager
         return new DiskBoundaries(cfs, dirs, positions, ringVersion, directoriesVersion);
     }
 
+    private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetadata tmd)
+    {
+        RangesAtEndpoint localRanges;
+        if (StorageService.instance.isBootstrapMode()
+        && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
+        {
+            PendingRangeCalculatorService.instance.blockUntilFinished();
+            localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
+        }
+        else
+        {
+            // Reason we use use the future settled TMD is that if we decommission a node, we want to stream
+            // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places.
+            // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled
+            localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort());
+        }
+        return localRanges;
+    }
+
     /**
      * Returns a list of disk boundaries, the result will differ depending on whether vnodes are enabled or not.
      *

src/java/org/apache/cassandra/db/Keyspace.java

@@ -377,7 +377,7 @@ public class Keyspace
         if (!ksm.params.replication.equals(replicationParams))
         {
             logger.debug("New replication settings for keyspace {} - invalidating disk boundary caches", ksm.name);
-            columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateDiskBoundaries);
+            columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateLocalRanges);
         }
         replicationParams = ksm.params.replication;
     }
@@ -406,7 +406,7 @@ public class Keyspace
     // disassociate a cfs from this keyspace instance.
     private void unloadCf(ColumnFamilyStore cfs, boolean dropData)
     {
-        cfs.forceBlockingFlush();
+        cfs.unloadCf();
         cfs.invalidate(true, dropData);
     }
 
@@ -677,11 +677,11 @@ public class Keyspace
         return replicationStrategy;
     }
 
-    public List<Future<?>> flush()
+    public List<Future<?>> flush(ColumnFamilyStore.FlushReason reason)
     {
         List<Future<?>> futures = new ArrayList<>(columnFamilyStores.size());
         for (ColumnFamilyStore cfs : columnFamilyStores.values())
-            futures.add(cfs.forceFlush());
+            futures.add(cfs.forceFlush(reason));
         return futures;
     }
 

src/java/org/apache/cassandra/db/Memtable.java

@@ -1,682 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
-import org.apache.cassandra.db.commitlog.IntervalSet;
-import org.apache.cassandra.db.filter.ClusteringIndexFilter;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.partitions.AbstractBTreePartition;
-import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.AtomicBTreePartition;
-import org.apache.cassandra.db.partitions.Partition;
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.IncludingExcludingBounds;
-import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableMultiWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.SchemaConstants;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.memory.HeapPool;
-import org.apache.cassandra.utils.memory.MemtableAllocator;
-import org.apache.cassandra.utils.memory.MemtableCleaner;
-import org.apache.cassandra.utils.memory.MemtablePool;
-import org.apache.cassandra.utils.memory.NativePool;
-import org.apache.cassandra.utils.memory.SlabPool;
-
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
-import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_COMPUTE_STEPS;
-import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE;
-
-public class Memtable implements Comparable<Memtable>
-{
-    private static final Logger logger = LoggerFactory.getLogger(Memtable.class);
-
-    public static final MemtablePool MEMORY_POOL = createMemtableAllocatorPool();
-
-    private static MemtablePool createMemtableAllocatorPool()
-    {
-        long heapLimit = DatabaseDescriptor.getMemtableHeapSpaceInMiB() << 20;
-        long offHeapLimit = DatabaseDescriptor.getMemtableOffheapSpaceInMiB() << 20;
-        final float cleaningThreshold = DatabaseDescriptor.getMemtableCleanupThreshold();
-        final MemtableCleaner cleaner = ColumnFamilyStore::flushLargestMemtable;
-        switch (DatabaseDescriptor.getMemtableAllocationType())
-        {
-            case unslabbed_heap_buffers_logged:
-                return new HeapPool.Logged(heapLimit, cleaningThreshold, cleaner);
-            case unslabbed_heap_buffers:
-                return new HeapPool(heapLimit, cleaningThreshold, cleaner);
-            case heap_buffers:
-                return new SlabPool(heapLimit, 0, cleaningThreshold, cleaner);
-            case offheap_buffers:
-                return new SlabPool(heapLimit, offHeapLimit, cleaningThreshold, cleaner);
-            case offheap_objects:
-                return new NativePool(heapLimit, offHeapLimit, cleaningThreshold, cleaner);
-            default:
-                throw new AssertionError();
-        }
-    }
-
-    private static final int ROW_OVERHEAD_HEAP_SIZE;
-    static
-    {
-        int userDefinedOverhead = MEMTABLE_OVERHEAD_SIZE.getInt(-1);
-        if (userDefinedOverhead > 0) ROW_OVERHEAD_HEAP_SIZE = userDefinedOverhead;
-        else ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(MEMTABLE_OVERHEAD_COMPUTE_STEPS.getInt());
-    }
-
-    private final MemtableAllocator allocator;
-    private final AtomicLong liveDataSize = new AtomicLong(0);
-    private final AtomicLong currentOperations = new AtomicLong(0);
-
-    // the write barrier for directing writes to this memtable or the next during a switch
-    private volatile OpOrder.Barrier writeBarrier;
-    // the precise upper bound of CommitLogPosition owned by this memtable
-    private volatile AtomicReference<CommitLogPosition> commitLogUpperBound;
-    // the precise lower bound of CommitLogPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
-    private AtomicReference<CommitLogPosition> commitLogLowerBound;
-
-    // The approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
-    // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
-    private final CommitLogPosition approximateCommitLogLowerBound = CommitLog.instance.getCurrentPosition();
-
-    public int compareTo(Memtable that)
-    {
-        return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound);
-    }
-
-    public static final class LastCommitLogPosition extends CommitLogPosition
-    {
-        public LastCommitLogPosition(CommitLogPosition copy)
-        {
-            super(copy.segmentId, copy.position);
-        }
-    }
-
-    // We index the memtable by PartitionPosition only for the purpose of being able
-    // to select key range using Token.KeyBound. However put() ensures that we
-    // actually only store DecoratedKey.
-    private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
-    public final ColumnFamilyStore cfs;
-    private final long creationNano = nanoTime();
-
-    // The smallest timestamp for all partitions stored in this memtable
-    private long minTimestamp = Long.MAX_VALUE;
-
-    // Record the comparator of the CFS at the creation of the memtable. This
-    // is only used when a user update the CF comparator, to know if the
-    // memtable was created with the new or old comparator.
-    public final ClusteringComparator initialComparator;
-
-    private final ColumnsCollector columnsCollector;
-    private final StatsCollector statsCollector = new StatsCollector();
-
-    // only to be used by init(), to setup the very first memtable for the cfs
-    public Memtable(AtomicReference<CommitLogPosition> commitLogLowerBound, ColumnFamilyStore cfs)
-    {
-        this.cfs = cfs;
-        this.commitLogLowerBound = commitLogLowerBound;
-        this.allocator = MEMORY_POOL.newAllocator(cfs);
-        this.initialComparator = cfs.metadata().comparator;
-        this.cfs.scheduleFlush();
-        this.columnsCollector = new ColumnsCollector(cfs.metadata().regularAndStaticColumns());
-    }
-
-    // ONLY to be used for testing, to create a mock Memtable
-    @VisibleForTesting
-    public Memtable(TableMetadata metadata)
-    {
-        this.initialComparator = metadata.comparator;
-        this.cfs = null;
-        this.allocator = null;
-        this.columnsCollector = new ColumnsCollector(metadata.regularAndStaticColumns());
-    }
-
-    public MemtableAllocator getAllocator()
-    {
-        return allocator;
-    }
-
-    public long getLiveDataSize()
-    {
-        return liveDataSize.get();
-    }
-
-    public long getOperations()
-    {
-        return currentOperations.get();
-    }
-
-    @VisibleForTesting
-    public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
-    {
-        assert this.writeBarrier == null;
-        this.commitLogUpperBound = commitLogUpperBound;
-        this.writeBarrier = writeBarrier;
-        allocator.setDiscarding();
-    }
-
-    void setDiscarded()
-    {
-        allocator.setDiscarded();
-    }
-
-    // decide if this memtable should take the write, or if it should go to the next memtable
-    public boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition)
-    {
-        // if the barrier hasn't been set yet, then this memtable is still taking ALL writes
-        OpOrder.Barrier barrier = this.writeBarrier;
-        if (barrier == null)
-            return true;
-        // if the barrier has been set, but is in the past, we are definitely destined for a future memtable
-        if (!barrier.isAfter(opGroup))
-            return false;
-        // if we aren't durable we are directed only by the barrier
-        if (commitLogPosition == null)
-            return true;
-        while (true)
-        {
-            // otherwise we check if we are in the past/future wrt the CL boundary;
-            // if the boundary hasn't been finalised yet, we simply update it to the max of
-            // its current value and ours; if it HAS been finalised, we simply accept its judgement
-            // this permits us to coordinate a safe boundary, as the boundary choice is made
-            // atomically wrt our max() maintenance, so an operation cannot sneak into the past
-            CommitLogPosition currentLast = commitLogUpperBound.get();
-            if (currentLast instanceof LastCommitLogPosition)
-                return currentLast.compareTo(commitLogPosition) >= 0;
-            if (currentLast != null && currentLast.compareTo(commitLogPosition) >= 0)
-                return true;
-            if (commitLogUpperBound.compareAndSet(currentLast, commitLogPosition))
-                return true;
-        }
-    }
-
-    public CommitLogPosition getCommitLogLowerBound()
-    {
-        return commitLogLowerBound.get();
-    }
-
-    public CommitLogPosition getCommitLogUpperBound()
-    {
-        return commitLogUpperBound.get();
-    }
-
-    public boolean isLive()
-    {
-        return allocator.isLive();
-    }
-
-    public boolean isClean()
-    {
-        return partitions.isEmpty();
-    }
-
-    public boolean mayContainDataBefore(CommitLogPosition position)
-    {
-        return approximateCommitLogLowerBound.compareTo(position) < 0;
-    }
-
-    /**
-     * @return true if this memtable is expired. Expiration time is determined by CF's memtable_flush_period_in_ms.
-     */
-    public boolean isExpired()
-    {
-        int period = cfs.metadata().params.memtableFlushPeriodInMs;
-        return period > 0 && (nanoTime() - creationNano >= TimeUnit.MILLISECONDS.toNanos(period));
-    }
-
-    /**
-     * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
-     * OpOrdering.
-     *
-     * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
-     */
-    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
-    {
-        AtomicBTreePartition previous = partitions.get(update.partitionKey());
-
-        long initialSize = 0;
-        if (previous == null)
-        {
-            final DecoratedKey cloneKey = allocator.clone(update.partitionKey(), opGroup);
-            AtomicBTreePartition empty = new AtomicBTreePartition(cfs.metadata, cloneKey, allocator);
-            // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent
-            previous = partitions.putIfAbsent(cloneKey, empty);
-            if (previous == null)
-            {
-                previous = empty;
-                // allocate the row overhead after the fact; this saves over allocating and having to free after, but
-                // means we can overshoot our declared limit.
-                int overhead = (int) (cloneKey.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE);
-                allocator.onHeap().allocate(overhead, opGroup);
-                initialSize = 8;
-            }
-        }
-
-        long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer);
-        minTimestamp = Math.min(minTimestamp, previous.stats().minTimestamp);
-        liveDataSize.addAndGet(initialSize + pair[0]);
-        columnsCollector.update(update.columns());
-        statsCollector.update(update.stats());
-        currentOperations.addAndGet(update.operationCount());
-        return pair[1];
-    }
-
-    public int partitionCount()
-    {
-        return partitions.size();
-    }
-
-    public List<FlushRunnable> flushRunnables(LifecycleTransaction txn)
-    {
-        return createFlushRunnables(txn);
-    }
-
-    private List<FlushRunnable> createFlushRunnables(LifecycleTransaction txn)
-    {
-        DiskBoundaries diskBoundaries = cfs.getDiskBoundaries();
-        List<PartitionPosition> boundaries = diskBoundaries.positions;
-        List<Directories.DataDirectory> locations = diskBoundaries.directories;
-        if (boundaries == null)
-            return Collections.singletonList(new FlushRunnable(txn));
-
-        List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
-        PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound();
-        try
-        {
-            for (int i = 0; i < boundaries.size(); i++)
-            {
-                PartitionPosition t = boundaries.get(i);
-                runnables.add(new FlushRunnable(rangeStart, t, locations.get(i), txn));
-                rangeStart = t;
-            }
-            return runnables;
-        }
-        catch (Throwable e)
-        {
-            throw Throwables.propagate(abortRunnables(runnables, e));
-        }
-    }
-
-    public Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
-    {
-        if (runnables != null)
-            for (FlushRunnable runnable : runnables)
-                t = runnable.writer.abort(t);
-        return t;
-    }
-
-    public String toString()
-    {
-        return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
-                             cfs.name, hashCode(), FBUtilities.prettyPrintMemory(liveDataSize.get()), currentOperations,
-                             100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
-    }
-
-    public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange)
-    {
-        AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
-
-        boolean startIsMin = keyRange.left.isMinimum();
-        boolean stopIsMin = keyRange.right.isMinimum();
-
-        boolean isBound = keyRange instanceof Bounds;
-        boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
-        boolean includeStop = isBound || keyRange instanceof Range;
-        Map<PartitionPosition, AtomicBTreePartition> subMap;
-        if (startIsMin)
-            subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop);
-        else
-            subMap = stopIsMin
-                   ? partitions.tailMap(keyRange.left, includeStart)
-                   : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop);
-
-        int minLocalDeletionTime = Integer.MAX_VALUE;
-
-        // avoid iterating over the memtable if we purge all tombstones
-        if (cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones())
-            minLocalDeletionTime = findMinLocalDeletionTime(subMap.entrySet().iterator());
-
-        final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter = subMap.entrySet().iterator();
-
-        return new MemtableUnfilteredPartitionIterator(cfs, iter, minLocalDeletionTime, columnFilter, dataRange);
-    }
-
-    private int findMinLocalDeletionTime(Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iterator)
-    {
-        int minLocalDeletionTime = Integer.MAX_VALUE;
-        while (iterator.hasNext())
-        {
-            Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iterator.next();
-            minLocalDeletionTime = Math.min(minLocalDeletionTime, entry.getValue().stats().minLocalDeletionTime);
-        }
-        return minLocalDeletionTime;
-    }
-
-    public Partition getPartition(DecoratedKey key)
-    {
-        return partitions.get(key);
-    }
-
-    public long getMinTimestamp()
-    {
-        return minTimestamp;
-    }
-
-    /**
-     * For testing only. Give this memtable too big a size to make it always fail flushing.
-     */
-    @VisibleForTesting
-    public void makeUnflushable()
-    {
-        liveDataSize.addAndGet((long) 1024 * 1024 * 1024 * 1024 * 1024);
-    }
-
-    class FlushRunnable implements Callable<SSTableMultiWriter>
-    {
-        private final long estimatedSize;
-        private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush;
-
-        private final boolean isBatchLogTable;
-        private final SSTableMultiWriter writer;
-
-        // keeping these to be able to log what we are actually flushing
-        private final PartitionPosition from;
-        private final PartitionPosition to;
-
-        FlushRunnable(PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn)
-        {
-            this(partitions.subMap(from, to), flushLocation, from, to, txn);
-        }
-
-        FlushRunnable(LifecycleTransaction txn)
-        {
-            this(partitions, null, null, null, txn);
-        }
-
-        FlushRunnable(ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn)
-        {
-            this.toFlush = toFlush;
-            this.from = from;
-            this.to = to;
-            long keySize = 0;
-            for (PartitionPosition key : toFlush.keySet())
-            {
-                //  make sure we don't write non-sensical keys
-                assert key instanceof DecoratedKey;
-                keySize += ((DecoratedKey) key).getKey().remaining();
-            }
-            estimatedSize = (long) ((keySize // index entries
-                                    + keySize // keys in data file
-                                    + liveDataSize.get()) // data
-                                    * 1.2); // bloom filter and row index overhead
-
-            this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
-
-            if (flushLocation == null)
-                writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
-            else
-                writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
-
-        }
-
-        protected Directories getDirectories()
-        {
-            return cfs.getDirectories();
-        }
-
-        private void writeSortedContents()
-        {
-            logger.info("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to);
-
-            boolean trackContention = logger.isTraceEnabled();
-            int heavilyContendedRowCount = 0;
-            // (we can't clear out the map as-we-go to free up memory,
-            //  since the memtable is being used for queries in the "pending flush" category)
-            for (AtomicBTreePartition partition : toFlush.values())
-            {
-                // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
-                // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
-                // we don't need to preserve tombstones for repair. So if both operation are in this
-                // memtable (which will almost always be the case if there is no ongoing failure), we can
-                // just skip the entry (CASSANDRA-4667).
-                if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
-                    continue;
-
-                if (trackContention && partition.useLock())
-                    heavilyContendedRowCount++;
-
-                if (!partition.isEmpty())
-                {
-                    try (UnfilteredRowIterator iter = partition.unfilteredIterator())
-                    {
-                        writer.append(iter);
-                    }
-                }
-            }
-
-            long bytesFlushed = writer.getFilePointer();
-            logger.info("Completed flushing {} ({}) for commitlog position {}",
-                         writer.getFilename(),
-                         FBUtilities.prettyPrintMemory(bytesFlushed),
-                         commitLogUpperBound);
-            // Update the metrics
-            cfs.metric.bytesFlushed.inc(bytesFlushed);
-
-            if (heavilyContendedRowCount > 0)
-                logger.trace("High update contention in {}/{} partitions of {} ", heavilyContendedRowCount, toFlush.size(), Memtable.this);
-        }
-
-        public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn,
-                                                    Descriptor descriptor,
-                                                    RegularAndStaticColumns columns,
-                                                    EncodingStats stats)
-        {
-            MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata().comparator)
-                    .commitLogIntervals(new IntervalSet<>(commitLogLowerBound.get(), commitLogUpperBound.get()));
-
-            return cfs.createSSTableMultiWriter(descriptor,
-                                                toFlush.size(),
-                                                ActiveRepairService.UNREPAIRED_SSTABLE,
-                                                ActiveRepairService.NO_PENDING_REPAIR,
-                                                false,
-                                                sstableMetadataCollector,
-                                                new SerializationHeader(true, cfs.metadata(), columns, stats), txn);
-        }
-
-        @Override
-        public SSTableMultiWriter call()
-        {
-            writeSortedContents();
-            return writer;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "Flush " + cfs.keyspace + '.' + cfs.name;
-        }
-    }
-
-    private static int estimateRowOverhead(final int count)
-    {
-        // calculate row overhead
-        try (final OpOrder.Group group = new OpOrder().start())
-        {
-            int rowOverhead;
-            MemtableAllocator allocator = MEMORY_POOL.newAllocator(null);
-            ConcurrentNavigableMap<PartitionPosition, Object> partitions = new ConcurrentSkipListMap<>();
-            final Object val = new Object();
-            for (int i = 0 ; i < count ; i++)
-                partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val);
-            double avgSize = ObjectSizes.measureDeep(partitions) / (double) count;
-            rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize));
-            rowOverhead -= ObjectSizes.measureDeep(new LongToken(0));
-            rowOverhead += AtomicBTreePartition.EMPTY_SIZE;
-            rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE;
-            allocator.setDiscarding();
-            allocator.setDiscarded();
-            return rowOverhead;
-        }
-    }
-
-    public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator
-    {
-        private final ColumnFamilyStore cfs;
-        private final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter;
-        private final int minLocalDeletionTime;
-        private final ColumnFilter columnFilter;
-        private final DataRange dataRange;
-
-        public MemtableUnfilteredPartitionIterator(ColumnFamilyStore cfs, Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter, int minLocalDeletionTime, ColumnFilter columnFilter, DataRange dataRange)
-        {
-            this.cfs = cfs;
-            this.iter = iter;
-            this.minLocalDeletionTime = minLocalDeletionTime;
-            this.columnFilter = columnFilter;
-            this.dataRange = dataRange;
-        }
-
-        public int getMinLocalDeletionTime()
-        {
-            return minLocalDeletionTime;
-        }
-
-        public TableMetadata metadata()
-        {
-            return cfs.metadata();
-        }
-
-        public boolean hasNext()
-        {
-            return iter.hasNext();
-        }
-
-        public UnfilteredRowIterator next()
-        {
-            Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iter.next();
-            // Actual stored key should be true DecoratedKey
-            assert entry.getKey() instanceof DecoratedKey;
-            DecoratedKey key = (DecoratedKey)entry.getKey();
-            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
-
-            return filter.getUnfilteredRowIterator(columnFilter, entry.getValue());
-        }
-    }
-
-    private static class ColumnsCollector
-    {
-        private final HashMap<ColumnMetadata, AtomicBoolean> predefined = new HashMap<>();
-        private final ConcurrentSkipListSet<ColumnMetadata> extra = new ConcurrentSkipListSet<>();
-        ColumnsCollector(RegularAndStaticColumns columns)
-        {
-            for (ColumnMetadata def : columns.statics)
-                predefined.put(def, new AtomicBoolean());
-            for (ColumnMetadata def : columns.regulars)
-                predefined.put(def, new AtomicBoolean());
-        }
-
-        public void update(RegularAndStaticColumns columns)
-        {
-            for (ColumnMetadata s : columns.statics)
-                update(s);
-            for (ColumnMetadata r : columns.regulars)
-                update(r);
-        }
-
-        private void update(ColumnMetadata definition)
-        {
-            AtomicBoolean present = predefined.get(definition);
-            if (present != null)
-            {
-                if (!present.get())
-                    present.set(true);
-            }
-            else
-            {
-                extra.add(definition);
-            }
-        }
-
-        public RegularAndStaticColumns get()
-        {
-            RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
-            for (Map.Entry<ColumnMetadata, AtomicBoolean> e : predefined.entrySet())
-                if (e.getValue().get())
-                    builder.add(e.getKey());
-            return builder.addAll(extra).build();
-        }
-    }
-
-    private static class StatsCollector
-    {
-        private final AtomicReference<EncodingStats> stats = new AtomicReference<>(EncodingStats.NO_STATS);
-
-        public void update(EncodingStats newStats)
-        {
-            while (true)
-            {
-                EncodingStats current = stats.get();
-                EncodingStats updated = current.mergeWith(newStats);
-                if (stats.compareAndSet(current, updated))
-                    return;
-            }
-        }
-
-        public EncodingStats get()
-        {
-            return stats.get();
-        }
-    }
-}

src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java

@@ -24,12 +24,17 @@ import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.db.virtual.VirtualTable;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.CachedPartition;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 import org.apache.cassandra.db.rows.BaseRowIterator;
 import org.apache.cassandra.db.transform.RTBoundValidator;
 import org.apache.cassandra.db.transform.Transformation;
@@ -42,7 +47,9 @@ import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.tracing.Tracing;
@@ -313,19 +320,19 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         InputCollector<UnfilteredPartitionIterator> inputCollector = iteratorsForRange(view, controller);
         try
         {
+            SSTableReadsListener readCountUpdater = newReadCountUpdater();
             for (Memtable memtable : view.memtables)
             {
                 @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
-                Memtable.MemtableUnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange());
-                controller.updateMinOldestUnrepairedTombstone(iter.getMinLocalDeletionTime());
+                UnfilteredPartitionIterator iter = memtable.partitionIterator(columnFilter(), dataRange(), readCountUpdater);
+                controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime());
                 inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
             }
 
-            SSTableReadsListener readCountUpdater = newReadCountUpdater();
             for (SSTableReader sstable : view.sstables)
             {
                 @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
-                UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), readCountUpdater);
+                UnfilteredPartitionIterator iter = sstable.partitionIterator(columnFilter(), dataRange(), readCountUpdater);
                 inputCollector.addSSTableIterator(sstable, RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false));
 
                 if (!sstable.isRepaired())

src/java/org/apache/cassandra/db/SimpleBuilders.java

@@ -421,6 +421,12 @@ public abstract class SimpleBuilders
             return this;
         }
 
+        public Row.SimpleBuilder deletePrevious()
+        {
+            builder.addRowDeletion(Row.Deletion.regular(new DeletionTime(timestamp - 1, nowInSec)));
+            return this;
+        }
+
         public Row.SimpleBuilder delete(String columnName)
         {
             return add(columnName, null);

src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.cache.RowCacheSentinel;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.filter.*;
 import org.apache.cassandra.db.lifecycle.*;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.db.transform.RTBoundValidator;
@@ -660,19 +661,19 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         InputCollector<UnfilteredRowIterator> inputCollector = iteratorsForPartition(view, controller);
         try
         {
+            SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
+
             for (Memtable memtable : view.memtables)
             {
-                Partition partition = memtable.getPartition(partitionKey());
-                if (partition == null)
+                @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
+                UnfilteredRowIterator iter = memtable.rowIterator(partitionKey(), filter.getSlices(metadata()), columnFilter(), filter.isReversed(), metricsCollector);
+                if (iter == null)
                     continue;
 
                 minTimestamp = Math.min(minTimestamp, memtable.getMinTimestamp());
 
-                @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
-                UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition);
-
                 // Memtable data is always considered unrepaired
-                controller.updateMinOldestUnrepairedTombstone(partition.stats().minLocalDeletionTime);
+                controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime());
                 inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false));
 
                 mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone,
@@ -695,8 +696,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
             int nonIntersectingSSTables = 0;
             int includedDueToTombstones = 0;
 
-            SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
-
             if (controller.isTrackingRepairedStatus())
                 Tracing.trace("Collecting data from sstables and tracking repaired status");
 
@@ -859,17 +858,14 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
 
         ImmutableBTreePartition result = null;
+        SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
 
         Tracing.trace("Merging memtable contents");
         for (Memtable memtable : view.memtables)
         {
-            Partition partition = memtable.getPartition(partitionKey());
-            if (partition == null)
-                continue;
-
-            try (UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition))
+            try (UnfilteredRowIterator iter = memtable.rowIterator(partitionKey, filter.getSlices(metadata()), columnFilter(), isReversed(), metricsCollector))
             {
-                if (iter.isEmpty())
+                if (iter == null)
                     continue;
 
                 result = add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false),
@@ -883,7 +879,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         /* add the SSTables on disk */
         view.sstables.sort(SSTableReader.maxTimestampDescending);
         // read sorted sstables
-        SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
         for (SSTableReader sstable : view.sstables)
         {
             // if we've already seen a partition tombstone with a timestamp greater

src/java/org/apache/cassandra/db/StorageHook.java

@@ -84,7 +84,7 @@ public interface StorageHook
                                                          boolean reversed,
                                                          SSTableReadsListener listener)
             {
-                return sstable.iterator(key, slices, selectedColumns, reversed, listener);
+                return sstable.rowIterator(key, slices, selectedColumns, reversed, listener);
             }
         };
     }

src/java/org/apache/cassandra/db/SystemKeyspace.java

@@ -923,7 +923,9 @@ public final class SystemKeyspace
 
             for (String cfname : cfnames)
             {
-                futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush());
+                futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME)
+                                    .getColumnFamilyStore(cfname)
+                                    .forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED));
             }
             FBUtilities.waitOnFutures(futures);
         }
@@ -1433,7 +1435,8 @@ public final class SystemKeyspace
 
     public static void flushPaxosRepairHistory()
     {
-        Schema.instance.getColumnFamilyStoreInstance(PaxosRepairHistoryTable.id).forceBlockingFlush();
+        Schema.instance.getColumnFamilyStoreInstance(PaxosRepairHistoryTable.id)
+                       .forceBlockingFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED);
     }
 
     public static PaxosRepairHistory loadPaxosRepairHistory(String keyspace, String table)

src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java

@@ -420,9 +420,20 @@ public abstract class AbstractCommitLogSegmentManager
                 else if (!flushes.containsKey(dirtyTableId))
                 {
                     final ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(dirtyTableId);
-                    // can safely call forceFlush here as we will only ever block (briefly) for other attempts to flush,
-                    // no deadlock possibility since switchLock removal
-                    flushes.put(dirtyTableId, force ? cfs.forceFlush() : cfs.forceFlush(maxCommitLogPosition));
+
+                    if (cfs.memtableWritesAreDurable())
+                    {
+                        // The memtable does not need this data to be preserved (we only wrote it for PITR and CDC)
+                        segment.markClean(dirtyTableId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition());
+                    }
+                    else
+                    {
+                        // can safely call forceFlush here as we will only ever block (briefly) for other attempts to flush,
+                        // no deadlock possibility since switchLock removal
+                        flushes.put(dirtyTableId, force
+                                                  ? cfs.forceFlush(ColumnFamilyStore.FlushReason.COMMITLOG_DIRTY)
+                                                  : cfs.forceFlush(maxCommitLogPosition));
+                    }
                 }
             }
         }

src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java

@@ -66,15 +66,17 @@ public class CommitLogArchiver
     final String restoreCommand;
     final String restoreDirectories;
     public long restorePointInTime;
+    public CommitLogPosition snapshotCommitLogPosition;
     public final TimeUnit precision;
 
     public CommitLogArchiver(String archiveCommand, String restoreCommand, String restoreDirectories,
-            long restorePointInTime, TimeUnit precision)
+            long restorePointInTime, CommitLogPosition snapshotCommitLogPosition, TimeUnit precision)
     {
         this.archiveCommand = archiveCommand;
         this.restoreCommand = restoreCommand;
         this.restoreDirectories = restoreDirectories;
         this.restorePointInTime = restorePointInTime;
+        this.snapshotCommitLogPosition = snapshotCommitLogPosition;
         this.precision = precision;
         executor = !Strings.isNullOrEmpty(archiveCommand)
                 ? executorFactory()
@@ -85,7 +87,7 @@ public class CommitLogArchiver
 
     public static CommitLogArchiver disabled()
     {
-        return new CommitLogArchiver(null, null, null, Long.MAX_VALUE, TimeUnit.MICROSECONDS);
+        return new CommitLogArchiver(null, null, null, Long.MAX_VALUE, CommitLogPosition.NONE, TimeUnit.MICROSECONDS);
     }
 
     public static CommitLogArchiver construct()
@@ -129,7 +131,27 @@ public class CommitLogArchiver
                 {
                     throw new RuntimeException("Unable to parse restore target time", e);
                 }
-                return new CommitLogArchiver(archiveCommand, restoreCommand, restoreDirectories, restorePointInTime, precision);
+
+                String snapshotPosition = commitlog_commands.getProperty("snapshot_commitlog_position");
+                CommitLogPosition snapshotCommitLogPosition;
+                try
+                {
+
+                    snapshotCommitLogPosition = Strings.isNullOrEmpty(snapshotPosition)
+                                                ? CommitLogPosition.NONE
+                                                : CommitLogPosition.serializer.fromString(snapshotPosition);
+                }
+                catch (ParseException | NumberFormatException e)
+                {
+                    throw new RuntimeException("Unable to parse snapshot commit log position", e);
+                }
+
+                return new CommitLogArchiver(archiveCommand,
+                                             restoreCommand,
+                                             restoreDirectories,
+                                             restorePointInTime,
+                                             snapshotCommitLogPosition,
+                                             precision);
             }
         }
         catch (IOException e)

src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java

@@ -18,8 +18,11 @@
 package org.apache.cassandra.db.commitlog;
 
 import java.io.IOException;
+import java.text.ParseException;
 import java.util.Comparator;
 
+import com.google.common.base.Strings;
+
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.ISerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -118,5 +121,20 @@ public class CommitLogPosition implements Comparable<CommitLogPosition>
         {
             return TypeSizes.sizeof(clsp.segmentId) + TypeSizes.sizeof(clsp.position);
         }
+
+        public CommitLogPosition fromString(String position) throws ParseException
+        {
+            if (Strings.isNullOrEmpty(position))
+                return NONE;
+            String[] parts = position.split(",");
+            if (parts.length != 2)
+                throw new ParseException("Commit log position must be given as <segment>,<position>", 0);
+            return new CommitLogPosition(Long.parseLong(parts[0].trim()), Integer.parseInt(parts[1].trim()));
+        }
+
+        public String toString(CommitLogPosition position)
+        {
+            return position == NONE ? "" : String.format("%d, %d", position.segmentId, position.position);
+        }
     }
 }

src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java

@@ -130,7 +130,41 @@ public class CommitLogReplayer implements CommitLogReadHandler
                 }
             }
 
-            IntervalSet<CommitLogPosition> filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt, localHostId);
+            IntervalSet<CommitLogPosition> filter;
+            final CommitLogPosition snapshotPosition = commitLog.archiver.snapshotCommitLogPosition;
+            if (snapshotPosition == CommitLogPosition.NONE)
+            {
+                // normal path: snapshot position is not explicitly specified, find it from sstables
+                if (!cfs.memtableWritesAreDurable())
+                {
+                    filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt, localHostId);
+                }
+                else
+                {
+                    if (commitLog.archiver.restorePointInTime == Long.MAX_VALUE)
+                    {
+                        // Normal restart, everything is persisted and restored by the memtable itself.
+                        filter = new IntervalSet<>(CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+                    }
+                    else
+                    {
+                        // Point-in-time restore with a persistent memtable. In this case user should have restored
+                        // the memtable from a snapshot and specified that snapshot's commit log position, reaching
+                        // the "else" path below.
+                        // If they haven't, do not filter any commit log data -- this supports a mode of operation where
+                        // the user deletes old archived commit log segments when a snapshot completes -- but issue a
+                        // message as this may be inefficient / not what the user wants.
+                        logger.info("Point-in-time restore on a persistent memtable started without a snapshot time. " +
+                                    "All commit log data will be replayed.");
+                        filter = IntervalSet.empty();
+                    }
+                }
+            }
+            else
+            {
+                // If the positions is specified, it must override whatever we calculate.
+                filter = new IntervalSet<>(CommitLogPosition.NONE, snapshotPosition);
+            }
             cfPersisted.put(cfs.metadata.id, filter);
         }
         CommitLogPosition globalPosition = firstNotCovered(cfPersisted.values());
@@ -212,12 +246,14 @@ public class CommitLogReplayer implements CommitLogReadHandler
             if (keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
                 flushingSystem = true;
 
-            futures.addAll(keyspace.flush());
+            futures.addAll(keyspace.flush(ColumnFamilyStore.FlushReason.STARTUP));
         }
 
         // also flush batchlog incase of any MV updates
         if (!flushingSystem)
-            futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceFlush());
+            futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME)
+                                .getColumnFamilyStore(SystemKeyspace.BATCHES)
+                                .forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED));
 
         FBUtilities.waitOnFutures(futures);
 

src/java/org/apache/cassandra/db/compaction/CompactionController.java

@@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.FileDataInput;
@@ -263,10 +263,9 @@ public class CompactionController extends AbstractCompactionController
 
         for (Memtable memtable : memtables)
         {
-            Partition partition = memtable.getPartition(key);
-            if (partition != null)
+            if (memtable.rowIterator(key) != null)
             {
-                minTimestampSeen = Math.min(minTimestampSeen, partition.stats().minTimestamp);
+                minTimestampSeen = Math.min(minTimestampSeen, memtable.getMinTimestamp());
                 hasTimestamp = true;
             }
         }

src/java/org/apache/cassandra/db/compaction/CompactionTask.java

@@ -122,7 +122,7 @@ public class CompactionTask extends AbstractCompactionTask
         if (DatabaseDescriptor.isSnapshotBeforeCompaction())
         {
             Instant creationTime = now();
-            cfs.snapshotWithoutFlush(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime);
+            cfs.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime);
         }
 
         try (CompactionController controller = getCompactionController(transaction.originals()))

src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java

@@ -143,7 +143,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im
     public static LifecycleTransaction offline(OperationType operationType, Iterable<SSTableReader> readers)
     {
         // if offline, for simplicity we just use a dummy tracker
-        Tracker dummy = new Tracker(null, false);
+        Tracker dummy = Tracker.newDummyTracker();
         dummy.addInitialSSTables(readers);
         dummy.apply(updateCompacting(emptySet(), readers));
         return new LifecycleTransaction(dummy, operationType, readers);
@@ -155,7 +155,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im
     @SuppressWarnings("resource") // log closed during postCleanup
     public static LifecycleTransaction offline(OperationType operationType)
     {
-        Tracker dummy = new Tracker(null, false);
+        Tracker dummy = Tracker.newDummyTracker();
         return new LifecycleTransaction(dummy, new LogTransaction(operationType, dummy), Collections.emptyList());
     }
 

src/java/org/apache/cassandra/db/lifecycle/Tracker.java

@@ -29,7 +29,7 @@ import com.google.common.collect.*;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.io.util.File;
 import org.slf4j.Logger;
@@ -74,17 +74,23 @@ public class Tracker
     public final boolean loadsstables;
 
     /**
+     * @param columnFamilyStore
      * @param memtable Initial Memtable. Can be null.
      * @param loadsstables true to indicate to load SSTables (TODO: remove as this is only accessed from 2i)
      */
-    public Tracker(Memtable memtable, boolean loadsstables)
+    public Tracker(ColumnFamilyStore columnFamilyStore, Memtable memtable, boolean loadsstables)
     {
-        this.cfstore = memtable != null ? memtable.cfs : null;
+        this.cfstore = columnFamilyStore;
         this.view = new AtomicReference<>();
         this.loadsstables = loadsstables;
         this.reset(memtable);
     }
 
+    public static Tracker newDummyTracker()
+    {
+        return new Tracker(null, null, false);
+    }
+
     public LifecycleTransaction tryModify(SSTableReader sstable, OperationType operationType)
     {
         return tryModify(singleton(sstable), operationType);

src/java/org/apache/cassandra/db/lifecycle/View.java

@@ -26,7 +26,7 @@ import com.google.common.base.Predicate;
 import com.google.common.collect.*;
 
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -169,7 +169,7 @@ public class View
         return sstables.isEmpty()
                && liveMemtables.size() <= 1
                && flushingMemtables.size() == 0
-               && (liveMemtables.size() == 0 || liveMemtables.get(0).getOperations() == 0);
+               && (liveMemtables.size() == 0 || liveMemtables.get(0).operationCount() == 0);
     }
 
     @Override

src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java

@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ImmediateExecutor;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.Promise;
+import org.apache.cassandra.utils.memory.HeapPool;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+import org.apache.cassandra.utils.memory.MemtableCleaner;
+import org.apache.cassandra.utils.memory.MemtablePool;
+import org.apache.cassandra.utils.memory.NativePool;
+import org.apache.cassandra.utils.memory.SlabPool;
+
+/**
+ * A memtable that uses memory tracked and maybe allocated via a MemtableAllocator from a MemtablePool.
+ * Provides methods of memory tracking and triggering flushes when the relevant limits are reached.
+ */
+public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithCommitlog
+{
+    private static final Logger logger = LoggerFactory.getLogger(AbstractAllocatorMemtable.class);
+
+    public static final MemtablePool MEMORY_POOL = AbstractAllocatorMemtable.createMemtableAllocatorPool();
+
+    protected final Owner owner;
+    protected final MemtableAllocator allocator;
+
+    // Record the comparator of the CFS at the creation of the memtable. This
+    // is only used when a user update the CF comparator, to know if the
+    // memtable was created with the new or old comparator.
+    protected final ClusteringComparator initialComparator;
+
+    private final long creationNano = Clock.Global.nanoTime();
+
+    private static MemtablePool createMemtableAllocatorPool()
+    {
+        long heapLimit = DatabaseDescriptor.getMemtableHeapSpaceInMiB() << 20;
+        long offHeapLimit = DatabaseDescriptor.getMemtableOffheapSpaceInMiB() << 20;
+        float memtableCleanupThreshold = DatabaseDescriptor.getMemtableCleanupThreshold();
+        MemtableCleaner cleaner = AbstractAllocatorMemtable::flushLargestMemtable;
+        switch (DatabaseDescriptor.getMemtableAllocationType())
+        {
+        case unslabbed_heap_buffers_logged:
+            return new HeapPool.Logged(heapLimit, memtableCleanupThreshold, cleaner);
+        case unslabbed_heap_buffers:
+            logger.debug("Memtables allocating with on-heap buffers");
+            return new HeapPool(heapLimit, memtableCleanupThreshold, cleaner);
+        case heap_buffers:
+            logger.debug("Memtables allocating with on-heap slabs");
+            return new SlabPool(heapLimit, 0, memtableCleanupThreshold, cleaner);
+        case offheap_buffers:
+            logger.debug("Memtables allocating with off-heap buffers");
+            return new SlabPool(heapLimit, offHeapLimit, memtableCleanupThreshold, cleaner);
+        case offheap_objects:
+            logger.debug("Memtables allocating with off-heap objects");
+            return new NativePool(heapLimit, offHeapLimit, memtableCleanupThreshold, cleaner);
+        default:
+            throw new AssertionError();
+        }
+    }
+
+    // only to be used by init(), to setup the very first memtable for the cfs
+    public AbstractAllocatorMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
+    {
+        super(metadataRef, commitLogLowerBound);
+        this.allocator = MEMORY_POOL.newAllocator(metadataRef.toString());
+        this.initialComparator = metadata.get().comparator;
+        this.owner = owner;
+        scheduleFlush();
+    }
+
+    public MemtableAllocator getAllocator()
+    {
+        return allocator;
+    }
+
+    @Override
+    public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason)
+    {
+        switch (reason)
+        {
+        case SCHEMA_CHANGE:
+            return initialComparator != metadata().comparator // If the CF comparator has changed, because our partitions reference the old one
+                   || metadata().params.memtable.factory() != factory(); // If a different type of memtable is requested
+        case OWNED_RANGES_CHANGE:
+            return false; // by default we don't use the local ranges, thus this has no effect
+        default:
+            return true;
+        }
+    }
+
+    public void metadataUpdated()
+    {
+        // We decided not to swap out this memtable, but if the flush period has changed we must schedule it for the
+        // new expiration time.
+        scheduleFlush();
+    }
+
+    public void localRangesUpdated()
+    {
+        // nothing to be done by default
+    }
+
+    public void performSnapshot(String snapshotName)
+    {
+        throw new AssertionError("performSnapshot must be implemented if shouldSwitch(SNAPSHOT) can return false.");
+    }
+
+    protected abstract Factory factory();
+
+    public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        super.switchOut(writeBarrier, commitLogUpperBound);
+        allocator.setDiscarding();
+    }
+
+    public void discard()
+    {
+        super.discard();
+        allocator.setDiscarded();
+    }
+
+    public String toString()
+    {
+        MemoryUsage usage = Memtable.getMemoryUsage(this);
+        return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %s)",
+                             metadata.get().name,
+                             hashCode(),
+                             FBUtilities.prettyPrintMemory(getLiveDataSize()),
+                             operationCount(),
+                             usage);
+    }
+
+    @Override
+    public void addMemoryUsageTo(MemoryUsage stats)
+    {
+        stats.ownershipRatioOnHeap += getAllocator().onHeap().ownershipRatio();
+        stats.ownershipRatioOffHeap += getAllocator().offHeap().ownershipRatio();
+        stats.ownsOnHeap += getAllocator().onHeap().owns();
+        stats.ownsOffHeap += getAllocator().offHeap().owns();
+    }
+
+    public void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup)
+    {
+        getAllocator().onHeap().allocate(additionalSpace, opGroup);
+    }
+
+    public void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup)
+    {
+        getAllocator().offHeap().allocate(additionalSpace, opGroup);
+    }
+
+    void scheduleFlush()
+    {
+        int period = metadata().params.memtableFlushPeriodInMs;
+        if (period > 0)
+            scheduleFlush(owner, period);
+    }
+
+    private static void scheduleFlush(Owner owner, int period)
+    {
+        logger.trace("scheduling flush in {} ms", period);
+        WrappedRunnable runnable = new WrappedRunnable()
+        {
+            protected void runMayThrow()
+            {
+                Memtable current = owner.getCurrentMemtable();
+                if (current instanceof AbstractAllocatorMemtable)
+                    ((AbstractAllocatorMemtable) current).flushIfPeriodExpired();
+            }
+        };
+        ScheduledExecutors.scheduledTasks.schedule(runnable, period, TimeUnit.MILLISECONDS);
+    }
+
+    private void flushIfPeriodExpired()
+    {
+        int period = metadata().params.memtableFlushPeriodInMs;
+        if (period > 0 && (Clock.Global.nanoTime() - creationNano >= TimeUnit.MILLISECONDS.toNanos(period)))
+        {
+            if (isClean())
+            {
+                // if we're still clean, instead of swapping just reschedule a flush for later
+                scheduleFlush(owner, period);
+            }
+            else
+            {
+                // we'll be rescheduled by the constructor of the Memtable.
+                owner.signalFlushRequired(AbstractAllocatorMemtable.this,
+                                          ColumnFamilyStore.FlushReason.MEMTABLE_PERIOD_EXPIRED);
+            }
+        }
+    }
+
+    /**
+     * Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
+     * queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
+     */
+    public static Future<Boolean> flushLargestMemtable()
+    {
+        float largestRatio = 0f;
+        AbstractAllocatorMemtable largestMemtable = null;
+        Memtable.MemoryUsage largestUsage = null;
+        float liveOnHeap = 0, liveOffHeap = 0;
+        // we take a reference to the current main memtable for the CF prior to snapping its ownership ratios
+        // to ensure we have some ordering guarantee for performing the switchMemtableIf(), i.e. we will only
+        // swap if the memtables we are measuring here haven't already been swapped by the time we try to swap them
+        for (Memtable currentMemtable : ColumnFamilyStore.activeMemtables())
+        {
+            if (!(currentMemtable instanceof AbstractAllocatorMemtable))
+                continue;
+            AbstractAllocatorMemtable current = (AbstractAllocatorMemtable) currentMemtable;
+
+            // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF,
+            // both on- and off-heap, and select the largest of the two ratios to weight this CF
+            MemoryUsage usage = Memtable.newMemoryUsage();
+            current.addMemoryUsageTo(usage);
+
+            for (Memtable indexMemtable : current.owner.getIndexMemtables())
+                if (indexMemtable instanceof AbstractAllocatorMemtable)
+                    indexMemtable.addMemoryUsageTo(usage);
+
+            float ratio = Math.max(usage.ownershipRatioOnHeap, usage.ownershipRatioOffHeap);
+            if (ratio > largestRatio)
+            {
+                largestMemtable = current;
+                largestUsage = usage;
+                largestRatio = ratio;
+            }
+
+            liveOnHeap += usage.ownershipRatioOnHeap;
+            liveOffHeap += usage.ownershipRatioOffHeap;
+        }
+
+        Promise<Boolean> returnFuture = new AsyncPromise<>();
+
+        if (largestMemtable != null)
+        {
+            float usedOnHeap = MEMORY_POOL.onHeap.usedRatio();
+            float usedOffHeap = MEMORY_POOL.offHeap.usedRatio();
+            float flushingOnHeap = MEMORY_POOL.onHeap.reclaimingRatio();
+            float flushingOffHeap = MEMORY_POOL.offHeap.reclaimingRatio();
+            logger.info("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}",
+                        largestMemtable.owner, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap),
+                        ratio(flushingOnHeap, flushingOffHeap), ratio(largestUsage.ownershipRatioOnHeap, largestUsage.ownershipRatioOffHeap));
+
+            Future<CommitLogPosition> flushFuture = largestMemtable.owner.signalFlushRequired(largestMemtable, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
+            flushFuture.addListener(() -> {
+                try
+                {
+                    flushFuture.get();
+                    returnFuture.trySuccess(true);
+                }
+                catch (Throwable t)
+                {
+                    returnFuture.tryFailure(t);
+                }
+            }, ImmediateExecutor.INSTANCE);
+        }
+        else
+        {
+            logger.debug("Flushing of largest memtable, not done, no memtable found");
+
+            returnFuture.trySuccess(false);
+        }
+
+        return returnFuture;
+    }
+
+    private static String ratio(float onHeap, float offHeap)
+    {
+        return String.format("%.2f/%.2f", onHeap, offHeap);
+    }
+}

src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java

@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+public abstract class AbstractMemtable implements Memtable
+{
+    protected final AtomicLong currentOperations = new AtomicLong(0);
+    protected final ColumnsCollector columnsCollector;
+    protected final StatsCollector statsCollector = new StatsCollector();
+    // The smallest timestamp for all partitions stored in this memtable
+    protected AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE);
+    // The smallest local deletion time for all partitions in this memtable
+    protected AtomicInteger minLocalDeletionTime = new AtomicInteger(Integer.MAX_VALUE);
+    // Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not
+    // correct for their usage.
+
+    protected TableMetadataRef metadata;
+
+    public AbstractMemtable(TableMetadataRef metadataRef)
+    {
+        this.metadata = metadataRef;
+        this.columnsCollector = new ColumnsCollector(metadata.get().regularAndStaticColumns());
+    }
+
+    public TableMetadata metadata()
+    {
+        return metadata.get();
+    }
+
+    public long operationCount()
+    {
+        return currentOperations.get();
+    }
+
+    public long getMinTimestamp()
+    {
+        return minTimestamp.get();
+    }
+
+    public int getMinLocalDeletionTime()
+    {
+        return minLocalDeletionTime.get();
+    }
+
+    protected static void updateMin(AtomicLong minTracker, long newValue)
+    {
+        while (true)
+        {
+            long existing = minTracker.get();
+            if (existing <= newValue)
+                break;
+            if (minTracker.compareAndSet(existing, newValue))
+                break;
+        }
+    }
+
+    protected static void updateMin(AtomicInteger minTracker, int newValue)
+    {
+        while (true)
+        {
+            int existing = minTracker.get();
+            if (existing <= newValue)
+                break;
+            if (minTracker.compareAndSet(existing, newValue))
+                break;
+        }
+    }
+
+    RegularAndStaticColumns columns()
+    {
+        return columnsCollector.get();
+    }
+
+    EncodingStats encodingStats()
+    {
+        return statsCollector.get();
+    }
+
+    protected static class ColumnsCollector
+    {
+        private final HashMap<ColumnMetadata, AtomicBoolean> predefined = new HashMap<>();
+        private final ConcurrentSkipListSet<ColumnMetadata> extra = new ConcurrentSkipListSet<>();
+
+        ColumnsCollector(RegularAndStaticColumns columns)
+        {
+            for (ColumnMetadata def : columns.statics)
+                predefined.put(def, new AtomicBoolean());
+            for (ColumnMetadata def : columns.regulars)
+                predefined.put(def, new AtomicBoolean());
+        }
+
+        public void update(RegularAndStaticColumns columns)
+        {
+            for (ColumnMetadata s : columns.statics)
+                update(s);
+            for (ColumnMetadata r : columns.regulars)
+                update(r);
+        }
+
+        public void update(ColumnsCollector other)
+        {
+            for (Map.Entry<ColumnMetadata, AtomicBoolean> v : other.predefined.entrySet())
+                if (v.getValue().get())
+                    update(v.getKey());
+
+            extra.addAll(other.extra);
+        }
+
+        private void update(ColumnMetadata definition)
+        {
+            AtomicBoolean present = predefined.get(definition);
+            if (present != null)
+            {
+                if (!present.get())
+                    present.set(true);
+            }
+            else
+            {
+                extra.add(definition);
+            }
+        }
+
+        /**
+         * Get the current state of the columns set.
+         *
+         * Note: If this is executed while mutations are still being performed on the table (e.g. to prepare
+         * an sstable for streaming when Memtable.Factory.streamFromMemtable() is true), the resulting view may be
+         * in a somewhat inconsistent state (it may include partial updates, as well as miss updates older than
+         * ones it does include).
+         */
+        public RegularAndStaticColumns get()
+        {
+            RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder();
+            for (Map.Entry<ColumnMetadata, AtomicBoolean> e : predefined.entrySet())
+                if (e.getValue().get())
+                    builder.add(e.getKey());
+            return builder.addAll(extra).build();
+        }
+    }
+
+    protected static class StatsCollector
+    {
+        private final AtomicReference<EncodingStats> stats = new AtomicReference<>(EncodingStats.NO_STATS);
+
+        public void update(EncodingStats newStats)
+        {
+            while (true)
+            {
+                EncodingStats current = stats.get();
+                EncodingStats updated = current.mergeWith(newStats);
+                if (stats.compareAndSet(current, updated))
+                    return;
+            }
+        }
+
+        public EncodingStats get()
+        {
+            return stats.get();
+        }
+    }
+
+    protected abstract class AbstractFlushablePartitionSet<P extends Partition> implements FlushablePartitionSet<P>
+    {
+        public long dataSize()
+        {
+            return getLiveDataSize();
+        }
+
+        public CommitLogPosition commitLogLowerBound()
+        {
+            return AbstractMemtable.this.getCommitLogLowerBound();
+        }
+
+        public CommitLogPosition commitLogUpperBound()
+        {
+            return AbstractMemtable.this.getCommitLogUpperBound();
+        }
+
+        public EncodingStats encodingStats()
+        {
+            return AbstractMemtable.this.encodingStats();
+        }
+
+        public RegularAndStaticColumns columns()
+        {
+            return AbstractMemtable.this.columns();
+        }
+    }
+}

src/java/org/apache/cassandra/db/memtable/AbstractMemtableWithCommitlog.java

@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+/**
+ * Memtable that uses a commit log for persistence. Provides methods of tracking the commit log positions covered by
+ * it and safely switching between memtables.
+ */
+public abstract class AbstractMemtableWithCommitlog extends AbstractMemtable
+{
+    // The approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
+    // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
+    private final CommitLogPosition approximateCommitLogLowerBound = CommitLog.instance.getCurrentPosition();
+    // the precise lower bound of CommitLogPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
+    private final AtomicReference<CommitLogPosition> commitLogLowerBound;
+    // the write barrier for directing writes to this memtable or the next during a switch
+    private volatile OpOrder.Barrier writeBarrier;
+    // the precise upper bound of CommitLogPosition owned by this memtable
+    private volatile AtomicReference<CommitLogPosition> commitLogUpperBound;
+
+    public AbstractMemtableWithCommitlog(TableMetadataRef metadataRef, AtomicReference<CommitLogPosition> commitLogLowerBound)
+    {
+        super(metadataRef);
+        this.commitLogLowerBound = commitLogLowerBound;
+    }
+
+    public CommitLogPosition getApproximateCommitLogLowerBound()
+    {
+        return approximateCommitLogLowerBound;
+    }
+
+    public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
+    {
+        // This can prepare the memtable data for deletion; it will still be used while the flush is proceeding.
+        // A setDiscarded call will follow.
+        assert this.writeBarrier == null;
+        this.writeBarrier = writeBarrier;
+        this.commitLogUpperBound = commitLogUpperBound;
+    }
+
+    public void discard()
+    {
+        assert writeBarrier != null : "Memtable must be switched out before being discarded.";
+    }
+
+    // decide if this memtable should take the write, or if it should go to the next memtable
+    @Override
+    public boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition)
+    {
+        // if the barrier hasn't been set yet, then this memtable is still the newest and is taking ALL writes.
+        OpOrder.Barrier barrier = this.writeBarrier;
+        if (barrier == null)
+            return true;
+        // Note that if this races with taking the barrier the opGroup and commit log position we were given must
+        // necessarily be before the barrier and any LastCommitLogPosition is set, thus this function will return true
+        // and no update to commitLogUpperBound is necessary.
+
+        // If the barrier has been set and issued, but is in the past, we are definitely destined for a future memtable.
+        // Because we issue the barrier after taking LastCommitLogPosition and mutations take their position after
+        // taking the opGroup, this condition also ensures the given commit log position is greater than the chosen
+        // upper bound.
+        if (!barrier.isAfter(opGroup))
+            return false;
+        // We are in the segment of time between the barrier is constructed (and the memtable is switched out)
+        // and the barrier is issued.
+        // if we aren't durable we are directed only by the barrier
+        if (commitLogPosition == null)
+            return true;
+        while (true)
+        {
+            // If the CL boundary has been set, the mutation can be accepted depending on whether it falls before it.
+            // However, if it has not been set, the old sstable must still accept writes but we must also ensure that
+            // their positions are accounted for in the boundary (as there may be a delay between taking the log
+            // position for the boundary and setting it where a mutation sneaks in).
+            // Thus, if the boundary hasn't been finalised yet, we simply update it to the max of its current value and
+            // ours; this permits us to coordinate a safe boundary, as the boundary choice is made atomically wrt our
+            // max() maintenance, so an operation cannot sneak into the past.
+            CommitLogPosition currentLast = commitLogUpperBound.get();
+            if (currentLast instanceof LastCommitLogPosition)
+                return currentLast.compareTo(commitLogPosition) >= 0;
+            if (currentLast != null && currentLast.compareTo(commitLogPosition) >= 0)
+                return true;
+            if (commitLogUpperBound.compareAndSet(currentLast, commitLogPosition))
+                return true;
+        }
+    }
+
+    public CommitLogPosition getCommitLogLowerBound()
+    {
+        return commitLogLowerBound.get();
+    }
+
+    public CommitLogPosition getCommitLogUpperBound()
+    {
+        return commitLogUpperBound.get();
+    }
+
+    public boolean mayContainDataBefore(CommitLogPosition position)
+    {
+        return approximateCommitLogLowerBound.compareTo(position) < 0;
+    }
+}

src/java/org/apache/cassandra/db/memtable/Flushing.java

@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.DiskBoundaries;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class Flushing
+{
+    private static final Logger logger = LoggerFactory.getLogger(Flushing.class);
+
+    private Flushing() // prevent instantiation
+    {
+    }
+
+    public static List<FlushRunnable> flushRunnables(ColumnFamilyStore cfs,
+                                                     Memtable memtable,
+                                                     LifecycleTransaction txn)
+    {
+        DiskBoundaries diskBoundaries = cfs.getDiskBoundaries();
+        List<PartitionPosition> boundaries = diskBoundaries.positions;
+        List<Directories.DataDirectory> locations = diskBoundaries.directories;
+        if (boundaries == null)
+        {
+            FlushRunnable runnable = flushRunnable(cfs, memtable, null, null, txn, null);
+            return Collections.singletonList(runnable);
+        }
+
+        List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
+        PartitionPosition rangeStart = boundaries.get(0).getPartitioner().getMinimumToken().minKeyBound();
+        try
+        {
+            for (int i = 0; i < boundaries.size(); i++)
+            {
+                PartitionPosition t = boundaries.get(i);
+                FlushRunnable runnable = flushRunnable(cfs, memtable, rangeStart, t, txn, locations.get(i));
+
+                runnables.add(runnable);
+                rangeStart = t;
+            }
+            return runnables;
+        }
+        catch (Throwable e)
+        {
+            throw Throwables.propagate(abortRunnables(runnables, e));
+        }
+    }
+
+    @SuppressWarnings("resource")   // writer owned by runnable, to be closed or aborted by its caller
+    static FlushRunnable flushRunnable(ColumnFamilyStore cfs,
+                                       Memtable memtable,
+                                       PartitionPosition from,
+                                       PartitionPosition to,
+                                       LifecycleTransaction txn,
+                                       Directories.DataDirectory flushLocation)
+    {
+        Memtable.FlushablePartitionSet<?> flushSet = memtable.getFlushSet(from, to);
+        SSTableFormat.Type formatType = SSTableFormat.Type.current();
+        long estimatedSize = formatType.info.getWriterFactory().estimateSize(flushSet);
+
+        Descriptor descriptor = flushLocation == null
+                                ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), formatType)
+                                : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), formatType);
+
+        SSTableMultiWriter writer = createFlushWriter(cfs,
+                                                      flushSet,
+                                                      txn,
+                                                      descriptor,
+                                                      flushSet.partitionCount());
+
+        return new FlushRunnable(flushSet, writer, cfs.metric, true);
+    }
+
+    public static Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
+    {
+        if (runnables != null)
+            for (FlushRunnable runnable : runnables)
+                t = runnable.writer.abort(t);
+        return t;
+    }
+
+    public static class FlushRunnable implements Callable<SSTableMultiWriter>
+    {
+        private final Memtable.FlushablePartitionSet<?> toFlush;
+
+        private final SSTableMultiWriter writer;
+        private final TableMetrics metrics;
+        private final boolean isBatchLogTable;
+        private final boolean logCompletion;
+
+        public FlushRunnable(Memtable.FlushablePartitionSet<?> flushSet,
+                             SSTableMultiWriter writer,
+                             TableMetrics metrics,
+                             boolean logCompletion)
+        {
+            this.toFlush = flushSet;
+            this.writer = writer;
+            this.metrics = metrics;
+            this.isBatchLogTable = toFlush.metadata() == SystemKeyspace.Batches;
+            this.logCompletion = logCompletion;
+        }
+
+        private void writeSortedContents()
+        {
+            logger.info("Writing {}, flushed range = [{}, {})", toFlush.memtable(), toFlush.from(), toFlush.to());
+
+            // (we can't clear out the map as-we-go to free up memory,
+            //  since the memtable is being used for queries in the "pending flush" category)
+            for (Partition partition : toFlush)
+            {
+                // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
+                // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
+                // we don't need to preserve tombstones for repair. So if both operation are in this
+                // memtable (which will almost always be the case if there is no ongoing failure), we can
+                // just skip the entry (CASSANDRA-4667).
+                if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
+                    continue;
+
+                if (!partition.isEmpty())
+                {
+                    try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+                    {
+                        writer.append(iter);
+                    }
+                }
+            }
+
+            if (logCompletion)
+            {
+                long bytesFlushed = writer.getFilePointer();
+                logger.info("Completed flushing {} ({}) for commitlog position {}",
+                            writer.getFilename(),
+                            FBUtilities.prettyPrintMemory(bytesFlushed),
+                            toFlush.memtable().getCommitLogUpperBound());
+                // Update the metrics
+                metrics.bytesFlushed.inc(bytesFlushed);
+            }
+        }
+
+        @Override
+        public SSTableMultiWriter call()
+        {
+            writeSortedContents();
+            return writer;
+            // We don't close the writer on error as the caller aborts all runnables if one happens.
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Flush " + toFlush.metadata().keyspace + '.' + toFlush.metadata().name;
+        }
+    }
+
+    public static SSTableMultiWriter createFlushWriter(ColumnFamilyStore cfs,
+                                                       Memtable.FlushablePartitionSet<?> flushSet,
+                                                       LifecycleTransaction txn,
+                                                       Descriptor descriptor,
+                                                       long partitionCount)
+    {
+        MetadataCollector sstableMetadataCollector = new MetadataCollector(flushSet.metadata().comparator)
+                                                     .commitLogIntervals(new IntervalSet<>(flushSet.commitLogLowerBound(),
+                                                                                           flushSet.commitLogUpperBound()));
+
+        return cfs.createSSTableMultiWriter(descriptor,
+                                            partitionCount,
+                                            ActiveRepairService.UNREPAIRED_SSTABLE,
+                                            ActiveRepairService.NO_PENDING_REPAIR,
+                                            false,
+                                            sstableMetadataCollector,
+                                            new SerializationHeader(true,
+                                                                    flushSet.metadata(),
+                                                                    flushSet.columns(),
+                                                                    flushSet.encodingStats()),
+                                            txn);
+    }
+}

src/java/org/apache/cassandra/db/memtable/Memtable.java

@@ -0,0 +1,431 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredSource;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+/**
+ * Memtable interface. This defines the operations the ColumnFamilyStore can perform with memtables.
+ * They are of several types:
+ * - construction factory interface
+ * - write and read operations: put, rowIterator and partitionIterator
+ * - statistics and features, including partition counts, data size, encoding stats, written columns
+ * - memory usage tracking, including methods of retrieval and of adding extra allocated space (used non-CFS secondary
+ *   indexes)
+ * - flush functionality, preparing the set of partitions to flush for given ranges
+ * - lifecycle management, i.e. operations that prepare and execute switch to a different memtable, together
+ *   with ways of tracking the affected commit log spans
+ *
+ * See Memtable_API.md for details on implementing and using alternative memtable implementations.
+ */
+public interface Memtable extends Comparable<Memtable>, UnfilteredSource
+{
+    // Construction
+
+    /**
+     * Factory interface for constructing memtables, and querying write durability features.
+     *
+     * The factory is chosen using the MemtableParams class (passed as argument to
+     * {@code CREATE TABLE ... WITH memtable = '<configuration_name>'} where the configuration definition is a map given
+     * under {@code memtable_configurations} in cassandra.yaml). To make that possible, implementations must provide
+     * either a static {@code FACTORY} field (if they accept no further option) or a static
+     * {@code factory(Map<String, String>)} method. In the latter case, the method should avoid creating
+     * multiple instances of the factory for the same parameters, or factories should at least implement hashCode and
+     * equals.
+     */
+    interface Factory
+    {
+        /**
+         * Create a memtable.
+         *
+         * @param commitLogLowerBound A commit log lower bound for the new memtable. This will be equal to the previous
+         *                            memtable's upper bound and defines the span of positions that any flushed sstable
+         *                            will cover.
+         * @param metadaRef Pointer to the up-to-date table metadata.
+         * @param owner Owning objects that will receive flush requests triggered by the memtable (e.g. on expiration).
+         */
+        Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadaRef, Owner owner);
+
+        /**
+         * If the memtable can achieve write durability directly (i.e. using some feature other than the commitlog, e.g.
+         * persistent memory), it can return true here, in which case the commit log will not store mutations in this
+         * table.
+         * Note that doing so will prevent point-in-time restores and changed data capture, thus a durable memtable must
+         * allow the option of turning commit log writing on even if it does not need it.
+         */
+        default boolean writesShouldSkipCommitLog()
+        {
+            return false;
+        }
+
+        /**
+         * This should be true if the memtable can achieve write durability for crash recovery directly (i.e. using some
+         * feature other than the commitlog, e.g. persistent memory).
+         * Setting this flag to true means that the commitlog should not replay mutations for this table on restart,
+         * and that it should not try to preserve segments that contain relevant data.
+         * Unless writesShouldSkipCommitLog() is also true, writes will be recorded in the commit log as they may be
+         * needed for changed data capture or point-in-time restore.
+         */
+        default boolean writesAreDurable()
+        {
+            return false;
+        }
+
+        /**
+         * Normally we can receive streamed sstables directly, skipping the memtable stage (zero-copy-streaming). When
+         * the memtable is the primary data store (e.g. persistent memtables), it will usually prefer to receive the
+         * data instead.
+         *
+         * If this returns true, all streamed sstables's content will be read and replayed as mutations, disabling
+         * zero-copy streaming.
+         */
+        default boolean streamToMemtable()
+        {
+            return false;
+        }
+
+        /**
+         * When we need to stream data, we usually flush and stream the resulting sstables. This will not work correctly
+         * if the memtable does not want to flush for streaming (e.g. persistent memtables acting as primary data
+         * store), because data (not just recent) will be missing from the streamed view. Such memtables must present
+         * their data separately for streaming.
+         * In other words if the memtable returns false on shouldSwitch(STREAMING/REPAIR), its factory must return true
+         * here.
+         *
+         * If this flag returns true, streaming will write the relevant content that resides in the memtable to
+         * temporary sstables, stream these sstables and then delete them.
+         */
+        default boolean streamFromMemtable()
+        {
+            return false;
+        }
+
+        /**
+         * Override this method to include implementation-specific memtable metrics in the table metrics.
+         *
+         * Memtable metrics lifecycle matches table lifecycle. It is the table that owns the metrics and
+         * decides when to release them.
+         */
+        default TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
+        {
+            return null;
+        }
+    }
+
+    /**
+     * Interface for providing signals back and requesting information from the owner, i.e. the object that controls the
+     * memtable. This is usually the ColumnFamilyStore; the interface is used to limit the dependency of memtables on
+     * the details of its implementation.
+     */
+    interface Owner
+    {
+        /** Signal to the owner that a flush is required (e.g. in response to hitting space limits) */
+        Future<CommitLogPosition> signalFlushRequired(Memtable memtable, ColumnFamilyStore.FlushReason reason);
+
+        /** Get the current memtable for this owner. Used to avoid capturing memtable in scheduled flush tasks. */
+        Memtable getCurrentMemtable();
+
+        /**
+         * Collect the index memtables flushed together with this. Used to accurately calculate memory that would be
+         * freed by a flush.
+         */
+        Iterable<Memtable> getIndexMemtables();
+
+        /**
+         * Construct a list of boundaries that split the locally-owned ranges into the given number of shards,
+         * splitting the owned space evenly. It is up to the memtable to use this information.
+         * Any changes in the ring structure (e.g. added or removed nodes) will invalidate the splits; in such a case
+         * the memtable will be sent a {@link #shouldSwitch}(OWNED_RANGES_CHANGE) and, should that return false, a
+         * {@link #localRangesUpdated()} call.
+         */
+        ShardBoundaries localRangeSplits(int shardCount);
+    }
+
+    // Main write and read operations
+
+    /**
+     * Put new data in the memtable. This operation may block until enough memory is available in the memory pool.
+     *
+     * @param update the partition update, may be a new partition or an update to an existing one
+     * @param indexer receives information about the update's effect
+     * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a
+     *                flush to free space.
+     *
+     * @return the smallest timestamp delta between corresponding rows from existing and update. A
+     * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition
+     * and those in {@code update}. See CASSANDRA-7979.
+     */
+    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup);
+
+    // Read operations are provided by the UnfilteredSource interface.
+
+    // Statistics
+
+    /** Number of partitions stored in the memtable */
+    long partitionCount();
+
+    /** Size of the data not accounting for any metadata / mapping overheads */
+    long getLiveDataSize();
+
+    /**
+     * Number of "operations" (in the sense defined in {@link PartitionUpdate#operationCount()}) the memtable has
+     * executed.
+     */
+    long operationCount();
+
+    /**
+     * The table's definition metadata.
+     *
+     * Note that this tracks the current state of the table and is not necessarily the same as what was used to create
+     * the memtable.
+     */
+    TableMetadata metadata();
+
+
+    // Memory usage tracking
+
+    /**
+     * Add this memtable's used memory to the given usage object. This can be used to retrieve a single memtable's usage
+     * as well as to combine the ones of related sstables (e.g. a table and its table-based secondary indexes).
+     */
+    void addMemoryUsageTo(MemoryUsage usage);
+
+
+    /**
+     * Creates a holder for memory usage collection.
+     *
+     * This is used to track on- and off-heap memory, as well as the ratio to the total permitted memtable memory.
+     */
+    static MemoryUsage newMemoryUsage()
+    {
+        return new MemoryUsage();
+    }
+
+    /**
+     * Shorthand for the getting a given table's memory usage.
+     * Implemented as a static to prevent implementations altering expectations by e.g. returning a cached object.
+     */
+    static MemoryUsage getMemoryUsage(Memtable memtable)
+    {
+        MemoryUsage usage = newMemoryUsage();
+        memtable.addMemoryUsageTo(usage);
+        return usage;
+    }
+
+    @NotThreadSafe
+    class MemoryUsage
+    {
+        /** On-heap memory used in bytes */
+        public long ownsOnHeap = 0;
+        /** Off-heap memory used in bytes */
+        public long ownsOffHeap = 0;
+        /** On-heap memory as ratio to permitted memtable space */
+        public float ownershipRatioOnHeap = 0.0f;
+        /** Off-heap memory as ratio to permitted memtable space */
+        public float ownershipRatioOffHeap = 0.0f;
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap",
+                                 FBUtilities.prettyPrintMemory(ownsOnHeap),
+                                 ownershipRatioOnHeap * 100,
+                                 FBUtilities.prettyPrintMemory(ownsOffHeap),
+                                 ownershipRatioOffHeap * 100);
+        }
+    }
+
+    /**
+     * Adjust the used on-heap space by the given size (e.g. to reflect memory used by a non-table-based index).
+     * This operation may block until enough memory is available in the memory pool.
+     *
+     * @param additionalSpace the number of allocated bytes
+     * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a
+     *                flush to free space.
+     */
+    void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup);
+
+    /**
+     * Adjust the used off-heap space by the given size (e.g. to reflect memory used by a non-table-based index).
+     * This operation may block until enough memory is available in the memory pool.
+     *
+     * @param additionalSpace the number of allocated bytes
+     * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a
+     *                flush to free space.
+     */
+    void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup);
+
+
+    // Flushing
+
+    /**
+     * Get the collection of data between the given partition boundaries in a form suitable for flushing.
+     */
+    FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPosition to);
+
+    /**
+     * A collection of partitions for flushing plus some information required for writing an sstable.
+     *
+     * Note that the listed entries must conform with the specified metadata. In particular, if the memtable is still
+     * being written to, care must be taken to not list newer items as they may violate the bounds collected by the
+     * encoding stats or refer to columns that don't exist in the collected columns set.
+     */
+    interface FlushablePartitionSet<P extends Partition> extends Iterable<P>, SSTableWriter.SSTableSizeParameters
+    {
+        Memtable memtable();
+
+        PartitionPosition from();
+        PartitionPosition to();
+
+        /** The commit log position at the time that this memtable was created */
+        CommitLogPosition commitLogLowerBound();
+        /** The commit log position at the time that this memtable was switched out */
+        CommitLogPosition commitLogUpperBound();
+
+        /** The set of all columns that have been written */
+        RegularAndStaticColumns columns();
+        /** Statistics required for writing an sstable efficiently */
+        EncodingStats encodingStats();
+
+        default TableMetadata metadata()
+        {
+            return memtable().metadata();
+        }
+
+        default boolean isEmpty()
+        {
+            return partitionCount() > 0;
+        }
+    }
+
+
+    // Lifecycle management
+
+    /**
+     * Called to tell the memtable that it is being switched out and will be flushed (or dropped) and discarded.
+     * Will be followed by a {@link #getFlushSet} call (if the table is not truncated or dropped), and a
+     * {@link #discard}.
+     *
+     * @param writeBarrier The barrier that will signal that all writes to this memtable have completed. That is, the
+     *                     point after which writes cannot be accepted by this memtable (it is permitted for writes
+     *                     before this barrier to go into the next; see {@link #accepts}).
+     * @param commitLogUpperBound The upper commit log position for this memtable. The value may be modified after this
+     *                            call and will match the next memtable's lower commit log bound.
+     */
+    void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound);
+
+    /**
+     * This memtable is no longer in use or required for outstanding flushes or operations.
+     * All held memory must be released.
+     */
+    void discard();
+
+    /**
+     * Decide if this memtable should take a write with the given parameters, or if the write should go to the next
+     * memtable. This enforces that no writes after the barrier set by {@link #switchOut} can be accepted, and
+     * is also used to define a shared commit log bound as the upper for this memtable and lower for the next.
+     */
+    boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition);
+
+    /** Approximate commit log lower bound, <= getCommitLogLowerBound, used as a time stamp for ordering */
+    CommitLogPosition getApproximateCommitLogLowerBound();
+
+    /** The commit log position at the time that this memtable was created */
+    CommitLogPosition getCommitLogLowerBound();
+
+    /** The commit log position at the time that this memtable was switched out */
+    CommitLogPosition getCommitLogUpperBound();
+
+    /** True if the memtable can contain any data that was written before the given commit log position */
+    boolean mayContainDataBefore(CommitLogPosition position);
+
+    /** True if the memtable contains no data */
+    boolean isClean();
+
+    /** Order memtables by time as reflected in the commit log position at time of construction */
+    default int compareTo(Memtable that)
+    {
+        return this.getApproximateCommitLogLowerBound().compareTo(that.getApproximateCommitLogLowerBound());
+    }
+
+    /**
+     * Decides whether the memtable should be switched/flushed for the passed reason.
+     * Normally this will return true, but e.g. persistent memtables may choose not to flush. Returning false will
+     * trigger further action for some reasons:
+     * - SCHEMA_CHANGE will be followed by metadataUpdated().
+     * - OWNED_RANGES_CHANGE will be followed by localRangesUpdated().
+     * - SNAPSHOT will be followed by performSnapshot().
+     * - STREAMING/REPAIR will be followed by creating a FlushSet for the streamed/repaired ranges. This data will be
+     *   used to create sstables, which will be streamed and then deleted.
+     * This will not be called to perform truncation or drop (in that case the memtable is unconditionally dropped),
+     * but a flush may nevertheless be requested in that case to prepare a snapshot.
+     */
+    boolean shouldSwitch(ColumnFamilyStore.FlushReason reason);
+
+    /**
+     * Called when the table's metadata is updated. The memtable's metadata reference now points to the new version.
+     * This will not be called if {@link #shouldSwitch)(SCHEMA_CHANGE) returns true, as the memtable will be swapped out
+     * instead.
+     */
+    void metadataUpdated();
+
+    /**
+     * Called when the known ranges have been updated and owner.localRangeSplits() may return different values.
+     * This will not be called if {@link #shouldSwitch)(OWNED_RANGES_CHANGE) returns true, as the memtable will be
+     * swapped out instead.
+     */
+    void localRangesUpdated();
+
+    /**
+     * If the memtable needs to do some special action for snapshots (e.g. because it is persistent and does not want
+     * to flush), it should return false on the above with reason SNAPSHOT and implement this method.
+     */
+    void performSnapshot(String snapshotName);
+
+    /**
+     * Special commit log position marker used in the upper bound marker setting process
+     * (see {@link org.apache.cassandra.db.ColumnFamilyStore#setCommitLogUpperBound} and {@link AbstractMemtable#accepts})
+     */
+    public static final class LastCommitLogPosition extends CommitLogPosition
+    {
+        public LastCommitLogPosition(CommitLogPosition copy)
+        {
+            super(copy.segmentId, copy.position);
+        }
+    }
+}

src/java/org/apache/cassandra/db/memtable/Memtable_API.md

@@ -0,0 +1,177 @@
+# Memtable API
+
+[CEP-11](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-11%3A+Pluggable+memtable+implementations) 
+/ [CASSANDRA-17034](https://issues.apache.org/jira/browse/CASSANDRA-17034)
+
+## Configuration specification
+
+Memtable types and options are specified using memtable "configurations", which specify an implementation class
+and its parameters. 
+
+The memtable configurations are defined in `cassandra.yaml`, using the following format:
+
+```yaml
+memtable:
+    configurations:
+        〈configuration name〉:
+          class_name: 〈class〉
+          inherits: 〈configuration name〉
+          parameters:
+            〈parameters〉
+```
+
+Configurations can copy the properties from others, including being full copies of another, which can be useful for
+easily remapping one name to another configuration.
+
+The default memtable configuration is named `default`. It can be overridden if the yaml specifies it (including
+using inheritance to copy another configuration), and it can be inherited, even if it is not explicitly defined in
+the yaml (e.g. to change some parameter but not the memtable class).
+
+Examples:
+
+```yaml
+memtable:
+    configurations:
+        more_shards:
+          inherits: default
+          parameters:
+             shards: 32
+```
+
+```yaml
+memtable:
+    configurations:
+        skiplist:
+          class_name: SkipListMemtable
+        sharded:
+          class_name: ShardedSkipListMemtable
+        default:
+          inherits: sharded
+```
+
+Note that the database will only validate the memtable class and its parameters when a configuration needs to be
+instantiated for a table.
+
+## Memtable selection
+
+Once a configuration has been defined, it can be used by specifying it in the `memtable` parameter of a `CREATE TABLE`
+or `ALTER TABLE` statement, for example:
+
+```
+CREATE TABLE ... WITH ... AND memtable = 'trie';
+```
+or
+```
+ALTER TABLE ... WITH memtable = 'skiplist';
+```
+
+If a memtable is not specified, the configuration `default` will be used. To reset a table to the default memtable,
+use
+```
+ALTER TABLE ... WITH memtable = 'default';
+```
+
+The memtable configuration selection is per table, i.e. it will be propagated to all nodes in the cluster. If some nodes
+do not have a definition for that configuration, cannot instantiate the class, or are still on a version of Cassandra
+before 4.1, they will reject the schema change. We therefore recommend using a separate `ALTER` statement to change a
+table's memtable implementation; upgrading all nodes to 4.1 or later is required to use the API.
+
+As additional safety when first deploying an alternative implementation to a production cluster, one may consider
+first deploying a remapped `default` configuration to all nodes in the cluster, switching the schema to reference
+it, and then changing the implementation by modifying the configuration one node at a time.
+
+For example, a remapped default can be specified with this:
+```yaml
+memtable:
+    configurations:
+        better_memtable:
+            inherits: default
+```
+selected via
+```
+ALTER TABLE production_table WITH memtable = 'better_memtable';
+```
+and later switched one node at a time to
+```yaml
+memtable:
+    configurations:
+        our_memtable:
+            class_name: ...
+        better_memtable:
+            inherits: our_memtable
+```
+
+## Memtable implementation
+
+A memtable implementation is an implementation of the `Memtable` interface. The main work of the class will be
+performed by the `put`, `rowIterator` and `partitionIterator` methods, used to write and read information to/from the
+memtable. In addition to this, the implementation must support retrieval of the content in a form suitable for flushing 
+(via `getFlushSet`), memory use and statistics tracking, mechanics for triggering a flush for reasons
+controlled by the memtable (e.g. exhaustion of the given memory allocation), and finally mechanisms for tracking the
+commit log spans covered by a memtable.
+
+Abstract classes that provide the latter parts of the functionality (expected to be shared by most
+implementations) are provided as the `AbstractMemtable` (statistics tracking), `AbstractMemtableWithCommitlog` (adds
+commit log span tracking) and `AbstractAllocatorMemtable` (adds memory management via the `Allocator` class, together
+with flush triggering on memory use and time interval expiration).
+
+The memtable API also gives the memtable some control over flushing and the functioning of the commit log. The former
+is there to permit memtables that operate long-term and/or can handle some events internally, without a need to flush.
+The latter enables memtables that have an internal durability mechanism, such as ones using persistent memory or a
+tightly integrated commit log (e.g. using the commit log buffers for memtable data storage).
+
+The memtable implementation must also provide a mechanism for memtable construction called a memtable "factory" 
+(the `Memtable.Factory` interface). Some features of the implementation may be needed before an instance is created or
+where the memtable instance is not accessible. To make working with them more straightforward, the following 
+memtable-controlled options are implemented on the factory:
+
+- `boolean writesAreDurable()` should return true if the memtable has its own persistence mechanism and does not want
+  the commitlog to provide persistence. In this case the commit log can still store the writes for changed-data-capture (CDC)
+  or point-in-time restore (PITR), but it need not keep them for replay until the memtable is flushed.
+- `boolean writesShouldSkipCommitLog()` should return true if the memtable does not want the commit log to store any of
+  its data. The expectation for this flag is that a persistent memtable will take a configuration parameter to turn this
+  option on to improve performance. Enabling this flag is not compatible with CDC or PITR.
+- `boolean streamToMemtable()` and `boolean streamFromMemtable()` should return true if the memtable is long-lived and 
+  cannot flush to facilitate streaming. In this case the streaming code will implement the process in a way that 
+  retrieves data in the memtable before sending, and applies received data in the memtable instead of directly creating 
+  an sstable.
+
+### Instantiation and configuration
+
+The memtables are instantiated by the factory, which is constructed via reflection on creating a `ColumnFamilyStore` or
+altering the table's configuration.
+
+Memtable classes must either contain a static `FACTORY` field (if they take no arguments other than class), or implement 
+a `factory(Map<String, String>)` method, which is called using the configuration `parameters`. For validation, the
+latter should consume any further options (using `map.remove`).
+
+The `MemtableParams` class will look for the specified class name (prefixed with `org.apache.cassandra.db.memtable.`
+if only a short name was given), then look for a `factory` method. If it finds one, it will call it with a copy of the 
+supplied parameters; if it does not, it will look for the `FACTORY` field and use its value if found. It will error out
+if the class was not found, if neither the method or field was found, or if the user supplied parameters that did not 
+get consumed.
+
+Because multiple configurations and tables may use the same parameters, it is expected that the factory method will
+store and reuse constructed factories to avoid wasting space for duplicate objects (this is typical for configuration 
+objects in Cassandra).
+
+At this time many of the configuration parameters for memtables are still configured using top-level parameters like
+`memtable_allocation_type` in `cassandra.yaml` and `memtable_flush_period_in_ms` in the table schema.
+
+
+### Sample implementation
+
+The API comes with a proof-of-concept implementation, a sharded skip-list memtable implemented by the 
+`ShardedSkipListMemtable` class. The difference between this and the default memtable is that the sharded version breaks 
+the token space served by the node into roughly equal regions and uses a separate skip-list for each shard. Sharding
+spreads the write concurrency among these independent skip lists, reducing congestion and can lead to significantly
+improved write throughput.
+
+This implementation takes two parameters, `shards` which specifies the number of shards to split into (by default, the
+number of CPU threads available to the process) and `serialize_writes`, which, if set to `true` causes writes to the
+memtable to be synchronized. The latter can be useful to minimize space and time wasted for unsuccesful lockless 
+partition modification where a new copy of the partition would be prepared but not used due to concurrent modification.
+Regardless of the setting, reads can always execute in parallel, including concurrently with writes.
+
+Please note that sharding cannot be used with non-hashing partitioners (i.e. `ByteOrderPartitioner` or 
+`OrderPreservingPartitioner`).

src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.memtable;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * Holds boundaries (tokens) used to map a particular token (so partition key) to a shard id.
+ * In practice, each keyspace has its associated boundaries, see {@link Keyspace}.
+ * <p>
+ * Technically, if we use {@code n} shards, this is a list of {@code n-1} tokens and each token {@code tk} gets assigned
+ * to the shard ID corresponding to the slot of the smallest token in the list that is greater to {@code tk}, or {@code n}
+ * if {@code tk} is bigger than any token in the list.
+ */
+public class ShardBoundaries
+{
+    private static final Token[] EMPTY_TOKEN_ARRAY = new Token[0];
+
+    // Special boundaries that map all tokens to one shard.
+    // These boundaries will be used in either of these cases:
+    // - there is only 1 shard configured
+    // - the default partitioner doesn't support splitting
+    // - the keyspace is local system keyspace
+    public static final ShardBoundaries NONE = new ShardBoundaries(EMPTY_TOKEN_ARRAY, -1);
+
+    private final Token[] boundaries;
+    public final long ringVersion;
+
+    @VisibleForTesting
+    public ShardBoundaries(Token[] boundaries, long ringVersion)
+    {
+        this.boundaries = boundaries;
+        this.ringVersion = ringVersion;
+    }
+
+    public ShardBoundaries(List<Token> boundaries, long ringVersion)
+    {
+        this(boundaries.toArray(EMPTY_TOKEN_ARRAY), ringVersion);
+    }
+
+    /**
+     * Computes the shard to use for the provided token.
+     */
+    public int getShardForToken(Token tk)
+    {
+        for (int i = 0; i < boundaries.length; i++)
+        {
+            if (tk.compareTo(boundaries[i]) < 0)
+                return i;
+        }
+        return boundaries.length;
+    }
+
+    /**
+     * Computes the shard to use for the provided key.
+     */
+    public int getShardForKey(PartitionPosition key)
+    {
+        // Boundaries are missing if the node is not sufficiently initialized yet
+        if (boundaries.length == 0)
+            return 0;
+
+        assert (key.getPartitioner() == DatabaseDescriptor.getPartitioner());
+        return getShardForToken(key.getToken());
+    }
+
+    /**
+     * The number of shards that this boundaries support, that is how many different shard ids {@link #getShardForToken} might
+     * possibly return.
+     *
+     * @return the number of shards supported by theses boundaries.
+     */
+    public int shardCount()
+    {
+        return boundaries.length + 1;
+    }
+
+    @Override
+    public String toString()
+    {
+        if (boundaries.length == 0)
+            return "shard 0: (min, max)";
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("shard 0: (min, ").append(boundaries[0]).append(") ");
+        for (int i = 0; i < boundaries.length - 1; i++)
+            sb.append("shard ").append(i+1).append(": (").append(boundaries[i]).append(", ").append(boundaries[i+1]).append("] ");
+        sb.append("shard ").append(boundaries.length).append(": (").append(boundaries[boundaries.length-1]).append(", max)");
+        return sb.toString();
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ShardBoundaries that = (ShardBoundaries) o;
+
+        return Arrays.equals(boundaries, that.boundaries);
+    }
+
+    public int hashCode()
+    {
+        return Arrays.hashCode(boundaries);
+    }
+}

src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java

@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.memtable;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Iterators;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+import org.github.jamm.Unmetered;
+
+/**
+ * A proof-of-concept sharded memtable implementation. This implementation splits the partition skip-list into several
+ * independent skip-lists each covering a roughly equal part of the token space served by this node. This reduces
+ * congestion of the skip-list from concurrent writes and can lead to improved write throughput.
+ *
+ * The implementation takes two parameters:
+ * - shards: the number of shards to split into.
+ * - serialize_writes: if false, each shard may serve multiple writes in parallel; if true, writes to each shard are
+ *   synchronized.
+ *
+ * Also see Memtable_API.md.
+ */
+public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
+{
+    private static final Logger logger = LoggerFactory.getLogger(ShardedSkipListMemtable.class);
+
+    public static final String SHARDS_OPTION = "shards";
+    public static final String LOCKING_OPTION = "serialize_writes";
+
+    // The boundaries for the keyspace as they were calculated when the memtable is created.
+    // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
+    // The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard
+    // for a given key, even if we race with the StorageService initialization or with topology changes.
+    @Unmetered
+    final ShardBoundaries boundaries;
+
+    /**
+     * Core-specific memtable regions. All writes must go through the specific core. The data structures used
+     * are concurrent-read safe, thus reads can be carried out from any thread.
+     */
+    final MemtableShard[] shards;
+
+    @VisibleForTesting
+    public static final String SHARD_COUNT_PROPERTY = "cassandra.memtable.shard.count";
+
+    // default shard count, used when a specific number of shards is not specified in the parameters
+    private static final int SHARD_COUNT = Integer.getInteger(SHARD_COUNT_PROPERTY, FBUtilities.getAvailableProcessors());
+
+    private final Factory factory;
+
+    // only to be used by init(), to setup the very first memtable for the cfs
+    ShardedSkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound,
+                            TableMetadataRef metadataRef,
+                            Owner owner,
+                            Integer shardCountOption,
+                            Factory factory)
+    {
+        super(commitLogLowerBound, metadataRef, owner);
+        int shardCount = shardCountOption != null ? shardCountOption : SHARD_COUNT;
+        this.boundaries = owner.localRangeSplits(shardCount);
+        this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef);
+        this.factory = factory;
+    }
+
+    private static MemtableShard[] generatePartitionShards(int splits,
+                                                           MemtableAllocator allocator,
+                                                           TableMetadataRef metadata)
+    {
+        MemtableShard[] partitionMapContainer = new MemtableShard[splits];
+        for (int i = 0; i < splits; i++)
+            partitionMapContainer[i] = new MemtableShard(metadata, allocator);
+
+        return partitionMapContainer;
+    }
+
+    public boolean isClean()
+    {
+        for (MemtableShard shard : shards)
+            if (!shard.isEmpty())
+                return false;
+        return true;
+    }
+
+    @Override
+    protected Memtable.Factory factory()
+    {
+        return factory;
+    }
+
+    /**
+     * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
+     * OpOrdering.
+     *
+     * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
+     */
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        DecoratedKey key = update.partitionKey();
+        MemtableShard shard = shards[boundaries.getShardForKey(key)];
+        return shard.put(key, update, indexer, opGroup);
+    }
+
+    /**
+     * Technically we should scatter gather on all the core threads because the size in following calls are not
+     * using volatile variables, but for metrics purpose this should be good enough.
+     */
+    @Override
+    public long getLiveDataSize()
+    {
+        long total = 0L;
+        for (MemtableShard shard : shards)
+            total += shard.liveDataSize();
+        return total;
+    }
+
+    @Override
+    public long operationCount()
+    {
+        long total = 0L;
+        for (MemtableShard shard : shards)
+            total += shard.currentOperations();
+        return total;
+    }
+
+    @Override
+    public long partitionCount()
+    {
+        int total = 0;
+        for (MemtableShard shard : shards)
+            total += shard.size();
+        return total;
+    }
+
+    @Override
+    public long getMinTimestamp()
+    {
+        long min = Long.MAX_VALUE;
+        for (MemtableShard shard : shards)
+            min =  Long.min(min, shard.minTimestamp());
+        return min;
+    }
+
+    @Override
+    public int getMinLocalDeletionTime()
+    {
+        int min = Integer.MAX_VALUE;
+        for (MemtableShard shard : shards)
+            min = Integer.min(min, shard.minLocalDeletionTime());
+        return min;
+    }
+
+    @Override
+    RegularAndStaticColumns columns()
+    {
+        for (MemtableShard shard : shards)
+            columnsCollector.update(shard.columnsCollector);
+        return columnsCollector.get();
+    }
+
+    @Override
+    EncodingStats encodingStats()
+    {
+        for (MemtableShard shard : shards)
+            statsCollector.update(shard.statsCollector.get());
+        return statsCollector.get();
+    }
+
+    @Override
+    public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
+                                                                 final DataRange dataRange,
+                                                                 SSTableReadsListener readsListener)
+    {
+        AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
+
+        PartitionPosition left = keyRange.left;
+        PartitionPosition right = keyRange.right;
+
+        boolean isBound = keyRange instanceof Bounds;
+        boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+        boolean includeStop = isBound || keyRange instanceof Range;
+
+        Iterator<AtomicBTreePartition> iterator = getPartitionIterator(left, includeStart, right, includeStop);
+
+        return new MemtableUnfilteredPartitionIterator(metadata(), iterator, columnFilter, dataRange);
+        // readsListener is ignored as it only accepts sstable signals
+    }
+
+    private Iterator<AtomicBTreePartition> getPartitionIterator(PartitionPosition left, boolean includeStart, PartitionPosition right, boolean includeStop)
+    {
+        int leftShard = left != null && !left.isMinimum() ? boundaries.getShardForKey(left) : 0;
+        int rightShard = right != null && !right.isMinimum() ? boundaries.getShardForKey(right) : boundaries.shardCount() - 1;
+        Iterator<AtomicBTreePartition> iterator;
+        if (leftShard == rightShard)
+            iterator = shards[leftShard].getPartitionsSubMap(left, includeStart, right, includeStop).values().iterator();
+        else
+        {
+            Iterator<AtomicBTreePartition>[] iters = new Iterator[rightShard - leftShard + 1];
+            int i = leftShard;
+            iters[0] = shards[leftShard].getPartitionsSubMap(left, includeStart, null, true).values().iterator();
+            for (++i; i < rightShard; ++i)
+                iters[i - leftShard] = shards[i].partitions.values().iterator();
+            iters[i - leftShard] = shards[i].getPartitionsSubMap(null, true, right, includeStop).values().iterator();
+            iterator = Iterators.concat(iters);
+        }
+        return iterator;
+    }
+
+    private Partition getPartition(DecoratedKey key)
+    {
+        int shardIndex = boundaries.getShardForKey(key);
+        return shards[shardIndex].partitions.get(key);
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+    {
+        Partition p = getPartition(key);
+        if (p == null)
+            return null;
+        else
+            return p.unfilteredIterator(selectedColumns, slices, reversed);
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key)
+    {
+        Partition p = getPartition(key);
+        return p != null ? p.unfilteredIterator() : null;
+    }
+
+    public FlushablePartitionSet<AtomicBTreePartition> getFlushSet(PartitionPosition from, PartitionPosition to)
+    {
+        long keySize = 0;
+        int keyCount = 0;
+
+        for (Iterator<AtomicBTreePartition> it = getPartitionIterator(from, true, to,false); it.hasNext();)
+        {
+            AtomicBTreePartition en = it.next();
+            keySize += en.partitionKey().getKey().remaining();
+            keyCount++;
+        }
+        long partitionKeySize = keySize;
+        int partitionCount = keyCount;
+        Iterator<AtomicBTreePartition> toFlush = getPartitionIterator(from, true, to,false);
+
+        return new AbstractFlushablePartitionSet<AtomicBTreePartition>()
+        {
+            public Memtable memtable()
+            {
+                return ShardedSkipListMemtable.this;
+            }
+
+            public PartitionPosition from()
+            {
+                return from;
+            }
+
+            public PartitionPosition to()
+            {
+                return to;
+            }
+
+            public long partitionCount()
+            {
+                return partitionCount;
+            }
+
+            public Iterator<AtomicBTreePartition> iterator()
+            {
+                return toFlush;
+            }
+
+            public long partitionKeysSize()
+            {
+                return partitionKeySize;
+            }
+        };
+    }
+
+    static class MemtableShard
+    {
+        // The following fields are volatile as we have to make sure that when we
+        // collect results from all sub-ranges, the thread accessing the value
+        // is guaranteed to see the changes to the values.
+
+        // The smallest timestamp for all partitions stored in this shard
+        private final AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE);
+        private final AtomicInteger minLocalDeletionTime = new AtomicInteger(Integer.MAX_VALUE);
+
+        private final AtomicLong liveDataSize = new AtomicLong(0);
+
+        private final AtomicLong currentOperations = new AtomicLong(0);
+
+        // We index the memtable by PartitionPosition only for the purpose of being able
+        // to select key range using Token.KeyBound. However put() ensures that we
+        // actually only store DecoratedKey.
+        private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
+
+        private final ColumnsCollector columnsCollector;
+
+        private final StatsCollector statsCollector;
+
+        @Unmetered  // total pool size should not be included in memtable's deep size
+        private final MemtableAllocator allocator;
+
+        private final TableMetadataRef metadata;
+
+        @VisibleForTesting
+        MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator)
+        {
+            this.columnsCollector = new ColumnsCollector(metadata.get().regularAndStaticColumns());
+            this.statsCollector = new StatsCollector();
+            this.allocator = allocator;
+            this.metadata = metadata;
+        }
+
+        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        {
+            AtomicBTreePartition previous = partitions.get(key);
+
+            long initialSize = 0;
+            if (previous == null)
+            {
+                final DecoratedKey cloneKey = allocator.clone(key, opGroup);
+                AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator);
+                // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent
+                previous = partitions.putIfAbsent(cloneKey, empty);
+                if (previous == null)
+                {
+                    previous = empty;
+                    // allocate the row overhead after the fact; this saves over allocating and having to free after, but
+                    // means we can overshoot our declared limit.
+                    int overhead = (int) (cloneKey.getToken().getHeapSize() + SkipListMemtable.ROW_OVERHEAD_HEAP_SIZE);
+                    allocator.onHeap().allocate(overhead, opGroup);
+                    initialSize = 8;
+                }
+            }
+
+            long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer);
+            updateMin(minTimestamp, update.stats().minTimestamp);
+            updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
+            liveDataSize.addAndGet(initialSize + pair[0]);
+            columnsCollector.update(update.columns());
+            statsCollector.update(update.stats());
+            currentOperations.addAndGet(update.operationCount());
+            return pair[1];
+        }
+
+        private Map<PartitionPosition, AtomicBTreePartition> getPartitionsSubMap(PartitionPosition left,
+                                                                                 boolean includeLeft,
+                                                                                 PartitionPosition right,
+                                                                                 boolean includeRight)
+        {
+            if (left != null && left.isMinimum())
+                left = null;
+            if (right != null && right.isMinimum())
+                right = null;
+
+            try
+            {
+                if (left == null)
+                    return right == null ? partitions : partitions.headMap(right, includeRight);
+                else
+                    return right == null
+                           ? partitions.tailMap(left, includeLeft)
+                           : partitions.subMap(left, includeLeft, right, includeRight);
+            }
+            catch (IllegalArgumentException e)
+            {
+                logger.error("Invalid range requested {} - {}", left, right);
+                throw e;
+            }
+        }
+
+        public boolean isEmpty()
+        {
+            return partitions.isEmpty();
+        }
+
+        public int size()
+        {
+            return partitions.size();
+        }
+
+        long minTimestamp()
+        {
+            return minTimestamp.get();
+        }
+
+        long liveDataSize()
+        {
+            return liveDataSize.get();
+        }
+
+        long currentOperations()
+        {
+            return currentOperations.get();
+        }
+
+        public int minLocalDeletionTime()
+        {
+            return minLocalDeletionTime.get();
+        }
+    }
+
+    public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
+    {
+        private final TableMetadata metadata;
+        private final Iterator<AtomicBTreePartition> iter;
+        private final ColumnFilter columnFilter;
+        private final DataRange dataRange;
+
+        public MemtableUnfilteredPartitionIterator(TableMetadata metadata, Iterator<AtomicBTreePartition> iterator, ColumnFilter columnFilter, DataRange dataRange)
+        {
+            this.metadata = metadata;
+            this.iter = iterator;
+            this.columnFilter = columnFilter;
+            this.dataRange = dataRange;
+        }
+
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        public boolean hasNext()
+        {
+            return iter.hasNext();
+        }
+
+        public UnfilteredRowIterator next()
+        {
+            AtomicBTreePartition entry = iter.next();
+            DecoratedKey key = entry.partitionKey();
+            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+
+            return filter.getUnfilteredRowIterator(columnFilter, entry);
+        }
+    }
+
+    static class Locking extends ShardedSkipListMemtable
+    {
+        Locking(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption, Factory factory)
+        {
+            super(commitLogLowerBound, metadataRef, owner, shardCountOption, factory);
+        }
+
+        /**
+         * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
+         * OpOrdering.
+         *
+         * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
+         */
+        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        {
+            DecoratedKey key = update.partitionKey();
+            MemtableShard shard = shards[boundaries.getShardForKey(key)];
+            synchronized (shard)
+            {
+                return shard.put(key, update, indexer, opGroup);
+            }
+        }
+
+    }
+
+    public static Factory factory(Map<String, String> optionsCopy)
+    {
+        String shardsString = optionsCopy.remove(SHARDS_OPTION);
+        Integer shardCount = shardsString != null ? Integer.parseInt(shardsString) : null;
+        boolean isLocking = Boolean.parseBoolean(optionsCopy.remove(LOCKING_OPTION));
+        return new Factory(shardCount, isLocking);
+    }
+
+    static class Factory implements Memtable.Factory
+    {
+        final Integer shardCount;
+        final boolean isLocking;
+
+        Factory(Integer shardCount, boolean isLocking)
+        {
+            this.shardCount = shardCount;
+            this.isLocking = isLocking;
+        }
+
+        public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
+                               TableMetadataRef metadataRef,
+                               Owner owner)
+        {
+            return isLocking
+                   ? new Locking(commitLogLowerBound, metadataRef, owner, shardCount, this)
+                   : new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount, this);
+        }
+
+        public boolean equals(Object o)
+        {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+            Factory factory = (Factory) o;
+            return Objects.equals(shardCount, factory.shardCount);
+        }
+
+        public int hashCode()
+        {
+            return Objects.hash(shardCount);
+        }
+    }
+}

src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java

@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.memtable;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.AbstractBTreePartition;
+import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.AtomicBTreePartition;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.IncludingExcludingBounds;
+import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.MemtableAllocator;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_COMPUTE_STEPS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE;
+
+public class SkipListMemtable extends AbstractAllocatorMemtable
+{
+    private static final Logger logger = LoggerFactory.getLogger(SkipListMemtable.class);
+
+    public static final Factory FACTORY = SkipListMemtableFactory.INSTANCE;
+
+    protected static final int ROW_OVERHEAD_HEAP_SIZE;
+    static
+    {
+        int userDefinedOverhead = MEMTABLE_OVERHEAD_SIZE.getInt(-1);
+        if (userDefinedOverhead > 0)
+            ROW_OVERHEAD_HEAP_SIZE = userDefinedOverhead;
+        else
+            ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(MEMTABLE_OVERHEAD_COMPUTE_STEPS.getInt());
+    }
+
+    // We index the memtable by PartitionPosition only for the purpose of being able
+    // to select key range using Token.KeyBound. However put() ensures that we
+    // actually only store DecoratedKey.
+    private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
+
+    private final AtomicLong liveDataSize = new AtomicLong(0);
+
+    protected SkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner)
+    {
+        super(commitLogLowerBound, metadataRef, owner);
+    }
+
+    @Override
+    protected Factory factory()
+    {
+        return FACTORY;
+    }
+
+    @Override
+    public boolean isClean()
+    {
+        return partitions.isEmpty();
+    }
+
+    /**
+     * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
+     * OpOrdering.
+     *
+     * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
+     */
+    @Override
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        AtomicBTreePartition previous = partitions.get(update.partitionKey());
+
+        long initialSize = 0;
+        if (previous == null)
+        {
+            final DecoratedKey cloneKey = allocator.clone(update.partitionKey(), opGroup);
+            AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator);
+            // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent
+            previous = partitions.putIfAbsent(cloneKey, empty);
+            if (previous == null)
+            {
+                previous = empty;
+                // allocate the row overhead after the fact; this saves over allocating and having to free after, but
+                // means we can overshoot our declared limit.
+                int overhead = (int) (cloneKey.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE);
+                allocator.onHeap().allocate(overhead, opGroup);
+                initialSize = 8;
+            }
+        }
+
+        long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer);
+        updateMin(minTimestamp, update.stats().minTimestamp);
+        updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
+        liveDataSize.addAndGet(initialSize + pair[0]);
+        columnsCollector.update(update.columns());
+        statsCollector.update(update.stats());
+        currentOperations.addAndGet(update.operationCount());
+        return pair[1];
+    }
+
+    @Override
+    public long partitionCount()
+    {
+        return partitions.size();
+    }
+
+    @Override
+    public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
+                                                                 final DataRange dataRange,
+                                                                 SSTableReadsListener readsListener)
+    {
+        AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
+
+        PartitionPosition left = keyRange.left;
+        PartitionPosition right = keyRange.right;
+
+        boolean isBound = keyRange instanceof Bounds;
+        boolean includeLeft = isBound || keyRange instanceof IncludingExcludingBounds;
+        boolean includeRight = isBound || keyRange instanceof Range;
+        Map<PartitionPosition, AtomicBTreePartition> subMap = getPartitionsSubMap(left,
+                                                                                  includeLeft,
+                                                                                  right,
+                                                                                  includeRight);
+
+        return new MemtableUnfilteredPartitionIterator(metadata.get(), subMap, columnFilter, dataRange);
+        // readsListener is ignored as it only accepts sstable signals
+    }
+
+    private Map<PartitionPosition, AtomicBTreePartition> getPartitionsSubMap(PartitionPosition left,
+                                                                             boolean includeLeft,
+                                                                             PartitionPosition right,
+                                                                             boolean includeRight)
+    {
+        if (left != null && left.isMinimum())
+            left = null;
+        if (right != null && right.isMinimum())
+            right = null;
+
+        try
+        {
+            if (left == null)
+                return right == null ? partitions : partitions.headMap(right, includeRight);
+            else
+                return right == null
+                       ? partitions.tailMap(left, includeLeft)
+                       : partitions.subMap(left, includeLeft, right, includeRight);
+        }
+        catch (IllegalArgumentException e)
+        {
+            logger.error("Invalid range requested {} - {}", left, right);
+            throw e;
+        }
+    }
+
+    Partition getPartition(DecoratedKey key)
+    {
+        return partitions.get(key);
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+    {
+        Partition p = getPartition(key);
+        if (p == null)
+            return null;
+        else
+            return p.unfilteredIterator(selectedColumns, slices, reversed);
+    }
+
+    @Override
+    public UnfilteredRowIterator rowIterator(DecoratedKey key)
+    {
+        Partition p = getPartition(key);
+        return p != null ? p.unfilteredIterator() : null;
+    }
+
+    private static int estimateRowOverhead(final int count)
+    {
+        // calculate row overhead
+        try (final OpOrder.Group group = new OpOrder().start())
+        {
+            int rowOverhead;
+            MemtableAllocator allocator = MEMORY_POOL.newAllocator("");
+            ConcurrentNavigableMap<PartitionPosition, Object> partitions = new ConcurrentSkipListMap<>();
+            final Object val = new Object();
+            for (int i = 0 ; i < count ; i++)
+                partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val);
+            double avgSize = ObjectSizes.measureDeep(partitions) / (double) count;
+            rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize));
+            rowOverhead -= ObjectSizes.measureDeep(new LongToken(0));
+            rowOverhead += AtomicBTreePartition.EMPTY_SIZE;
+            rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE;
+            allocator.setDiscarding();
+            allocator.setDiscarded();
+            return rowOverhead;
+        }
+    }
+
+    @Override
+    public FlushablePartitionSet<?> getFlushSet(PartitionPosition from, PartitionPosition to)
+    {
+        Map<PartitionPosition, AtomicBTreePartition> toFlush = getPartitionsSubMap(from, true, to, false);
+        long keysSize = 0;
+        long keyCount = 0;
+
+        boolean trackContention = logger.isTraceEnabled();
+        if (trackContention)
+        {
+            int heavilyContendedRowCount = 0;
+
+            for (AtomicBTreePartition partition : toFlush.values())
+            {
+                keysSize += partition.partitionKey().getKey().remaining();
+                ++keyCount;
+                if (partition.useLock())
+                    heavilyContendedRowCount++;
+            }
+
+            if (heavilyContendedRowCount > 0)
+                logger.trace("High update contention in {}/{} partitions of {} ", heavilyContendedRowCount, toFlush.size(), SkipListMemtable.this);
+        }
+        else
+        {
+            for (PartitionPosition key : toFlush.keySet())
+            {
+                //  make sure we don't write non-sensical keys
+                assert key instanceof DecoratedKey;
+                keysSize += ((DecoratedKey) key).getKey().remaining();
+                ++keyCount;
+            }
+        }
+        final long partitionKeysSize = keysSize;
+        final long partitionCount = keyCount;
+
+        return new AbstractFlushablePartitionSet<AtomicBTreePartition>()
+        {
+            @Override
+            public Memtable memtable()
+            {
+                return SkipListMemtable.this;
+            }
+
+            @Override
+            public PartitionPosition from()
+            {
+                return from;
+            }
+
+            @Override
+            public PartitionPosition to()
+            {
+                return to;
+            }
+
+            @Override
+            public long partitionCount()
+            {
+                return partitionCount;
+            }
+
+            @Override
+            public Iterator<AtomicBTreePartition> iterator()
+            {
+                return toFlush.values().iterator();
+            }
+
+            @Override
+            public long partitionKeysSize()
+            {
+                return partitionKeysSize;
+            }
+        };
+    }
+
+
+    private static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
+    {
+        private final TableMetadata metadata;
+        private final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter;
+        private final ColumnFilter columnFilter;
+        private final DataRange dataRange;
+
+        MemtableUnfilteredPartitionIterator(TableMetadata metadata, Map<PartitionPosition, AtomicBTreePartition> map, ColumnFilter columnFilter, DataRange dataRange)
+        {
+            this.metadata = metadata;
+            this.iter = map.entrySet().iterator();
+            this.columnFilter = columnFilter;
+            this.dataRange = dataRange;
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return iter.hasNext();
+        }
+
+        @Override
+        public UnfilteredRowIterator next()
+        {
+            Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iter.next();
+            // Actual stored key should be true DecoratedKey
+            assert entry.getKey() instanceof DecoratedKey;
+            DecoratedKey key = (DecoratedKey)entry.getKey();
+            ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
+
+            return filter.getUnfilteredRowIterator(columnFilter, entry.getValue());
+        }
+    }
+
+    @Override
+    public long getLiveDataSize()
+    {
+        return liveDataSize.get();
+    }
+
+    /**
+     * For testing only. Give this memtable too big a size to make it always fail flushing.
+     */
+    @VisibleForTesting
+    public void makeUnflushable()
+    {
+        liveDataSize.addAndGet(1024L * 1024 * 1024 * 1024 * 1024);
+    }
+}

src/java/org/apache/cassandra/db/memtable/SkipListMemtableFactory.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.memtable;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.config.InheritingClass;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.schema.TableMetadataRef;
+
+/**
+ * This class makes better sense as an inner class to SkipListMemtable (which could be as simple as
+ * FACTORY = SkipListMemtable::new), but having it there causes the SkipListMemtable class to be initialized the first
+ * time it is referenced (e.g. during default memtable factory construction).
+ *
+ * Some tests want to setup table parameters before initializing DatabaseDescriptor -- this allows them to do so, and
+ * also makes sure the memtable memory pools are not created for offline tools.
+ */
+public class SkipListMemtableFactory implements Memtable.Factory
+{
+    @Override
+    public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadaRef, Memtable.Owner owner)
+    {
+        return new SkipListMemtable(commitLogLowerBound, metadaRef, owner);
+    }
+
+    public static final SkipListMemtableFactory INSTANCE = new SkipListMemtableFactory();
+    public static InheritingClass CONFIGURATION = new InheritingClass(null, SkipListMemtable.class.getName(), ImmutableMap.of());
+}

src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java

@@ -66,6 +66,11 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
             this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow;
             this.stats = stats;
         }
+
+        protected Holder withColumns(RegularAndStaticColumns columns)
+        {
+            return new Holder(columns, this.tree, this.deletionInfo, this.staticRow, this.stats);
+        }
     }
 
     public DeletionInfo deletionInfo()

src/java/org/apache/cassandra/db/partitions/Partition.java

@@ -49,6 +49,11 @@ public interface Partition
      */
     public boolean isEmpty();
 
+    /**
+     * Whether the partition object has rows. This may be false but partition still be non-empty if it has a deletion.
+     */
+    boolean hasRows();
+
     /**
      * Returns the row corresponding to the provided clustering, or null if there is not such row.
      *

src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java

@@ -21,10 +21,11 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
@@ -210,6 +211,20 @@ public class PartitionUpdate extends AbstractBTreePartition
         return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
     }
 
+
+    public PartitionUpdate withOnlyPresentColumns()
+    {
+        Set<ColumnMetadata> columnSet = new HashSet<>();
+
+        for (Row row : this)
+            for (ColumnData column : row)
+                columnSet.add(column.column());
+
+        RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build();
+        return new PartitionUpdate(this.metadata, this.partitionKey, this.holder.withColumns(columns), this.deletionInfo.mutableCopy(), false);
+    }
+
+
     protected boolean canHaveShadowedData()
     {
         return canHaveShadowedData;

src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java

@@ -29,6 +29,7 @@ import java.util.function.LongPredicate;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Maps;
 
 import org.slf4j.Logger;
@@ -56,7 +57,6 @@ import org.apache.cassandra.repair.ValidationPartitionIterator;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.repair.NoSuchRepairSessionException;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Refs;
 
@@ -196,12 +196,19 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
             if (!isIncremental)
             {
                 // flush first so everyone is validating data that is as similar as possible
-                StorageService.instance.forceKeyspaceFlush(cfs.keyspace.getName(), cfs.name);
+                cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION);
+                // Note: we also flush for incremental repair during the anti-compaction process.
             }
             sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental);
         }
 
+        // Persistent memtables will not flush or snapshot to sstables, make an sstable with their data.
+        cfs.writeAndAddMemtableRanges(parentId,
+                                      () -> Collections2.transform(Range.normalize(ranges), Range::makeRowRange),
+                                      sstables);
+
         Preconditions.checkArgument(sstables != null);
+
         ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId);
         logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}",
                     prs.previewKind.logPrefix(sessionID),

src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java

@@ -365,7 +365,7 @@ public class PendingAntiCompaction
         List<FutureTask<AcquireResult>> tasks = new ArrayList<>(tables.size());
         for (ColumnFamilyStore cfs : tables)
         {
-            cfs.forceBlockingFlush();
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.ANTICOMPACTION);
             FutureTask<AcquireResult> task = new FutureTask<>(getAcquisitionCallable(cfs, tokenRanges.ranges(), prsId, acquireRetrySeconds, acquireSleepMillis));
             executor.submit(task);
             tasks.add(task);

src/java/org/apache/cassandra/db/rows/Row.java

@@ -626,6 +626,14 @@ public interface Row extends Unfiltered, Iterable<ColumnData>, IMeasurableMemory
          */
         public SimpleBuilder delete();
 
+        /**
+         * Deletes the whole row with a timestamp that is just before the new data's timestamp, to make sure no expired
+         * data remains on the row.
+         *
+         * @return this builder.
+         */
+        public SimpleBuilder deletePrevious();
+
         /**
          * Removes the value for a given column (creating a tombstone).
          *

src/java/org/apache/cassandra/db/rows/Rows.java

@@ -273,7 +273,7 @@ public abstract class Rows
      *
      * @return the smallest timestamp delta between corresponding rows from existing and update. A
      * timestamp delta being computed as the difference between the cells and DeletionTimes from {@code existing}
-     * and those in {@code existing}.
+     * and those in {@code update}.
      */
     public static long merge(Row existing,
                              Row update,

src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java

@@ -97,7 +97,7 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt
     {
         @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
         UnfilteredRowIterator iter = RTBoundValidator.validate(
-            sstable.iterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener),
+            sstable.rowIterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener),
             RTBoundValidator.Stage.SSTABLE,
             false
         );

src/java/org/apache/cassandra/db/rows/UnfilteredSource.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+
+/**
+ * Common data access interface for sstables and memtables.
+ */
+public interface UnfilteredSource
+{
+    /**
+     * Returns a row iterator for the given partition, applying the specified row and column filters.
+     *
+     * @param key the partition key
+     * @param slices the row ranges to return
+     * @param columnFilter filter to apply to all returned partitions
+     * @param reversed true if the content should be returned in reverse order
+     * @param listener a listener used to handle internal read events
+     */
+    UnfilteredRowIterator rowIterator(DecoratedKey key,
+                                      Slices slices,
+                                      ColumnFilter columnFilter,
+                                      boolean reversed,
+                                      SSTableReadsListener listener);
+
+    default UnfilteredRowIterator rowIterator(DecoratedKey key)
+    {
+        return rowIterator(key, Slices.ALL, ColumnFilter.NONE, false, SSTableReadsListener.NOOP_LISTENER);
+    }
+
+    /**
+     * Returns a partition iterator for the given data range.
+     *
+     * @param columnFilter filter to apply to all returned partitions
+     * @param dataRange the partition and clustering range queried
+     * @param listener a listener used to handle internal read events
+     */
+    UnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter,
+                                                  DataRange dataRange,
+                                                  SSTableReadsListener listener);
+
+    /** Minimum timestamp of all stored data */
+    long getMinTimestamp();
+
+    /** Minimum local deletion time in the memtable */
+    int getMinLocalDeletionTime();
+}

src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java

@@ -81,6 +81,7 @@ public class CassandraStreamManager implements TableStreamManager
         return new CassandraStreamReceiver(cfs, session, totalStreams);
     }
 
+    @SuppressWarnings("resource")   // references placed onto returned collection or closed on error
     @Override
     public Collection<OutgoingStream> createOutgoingStreams(StreamSession session, RangesAtEndpoint replicas, TimeUUID pendingRepair, PreviewKind previewKind)
     {
@@ -126,6 +127,9 @@ public class CassandraStreamManager implements TableStreamManager
                 return sstables;
             }).refs);
 
+            // This call is normally preceded by a memtable flush in StreamSession.addTransferRanges.
+            // Persistent memtables will not flush, make an sstable with their data.
+            cfs.writeAndAddMemtableRanges(session.getPendingRepair(), () -> Range.normalize(keyRanges), refs);
 
             List<Range<Token>> normalizedFullRanges = Range.normalize(replicas.onlyFull().ranges());
             List<Range<Token>> normalizedAllRanges = Range.normalize(replicas.ranges());

src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java

@@ -181,11 +181,13 @@ public class CassandraStreamReceiver implements StreamReceiver
      * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they
      * can be archived by the CDC process on discard.
      */
-    private boolean requiresWritePath(ColumnFamilyStore cfs) {
-        return hasCDC(cfs) || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
+    private boolean requiresWritePath(ColumnFamilyStore cfs)
+    {
+        return hasCDC(cfs) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs));
     }
 
-    private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReader> readers) {
+    private void sendThroughWritePath(ColumnFamilyStore cfs, Collection<SSTableReader> readers)
+    {
         boolean hasCdc = hasCDC(cfs);
         ColumnFilter filter = ColumnFilter.all(cfs.metadata());
         for (SSTableReader reader : readers)
@@ -273,7 +275,7 @@ public class CassandraStreamReceiver implements StreamReceiver
         // the streamed sstables.
         if (requiresWritePath)
         {
-            cfs.forceBlockingFlush();
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.STREAMS_RECEIVED);
             abort();
         }
     }

src/java/org/apache/cassandra/db/view/TableViews.java

@@ -104,10 +104,10 @@ public class TableViews extends AbstractCollection<View>
         views.forEach(View::stopBuild);
     }
 
-    public void forceBlockingFlush()
+    public void forceBlockingFlush(ColumnFamilyStore.FlushReason reason)
     {
         for (ColumnFamilyStore viewCfs : allViewsCfs())
-            viewCfs.forceBlockingFlush();
+            viewCfs.forceBlockingFlush(reason);
     }
 
     public void dumpMemtables()

src/java/org/apache/cassandra/db/view/ViewBuilder.java

@@ -95,7 +95,7 @@ class ViewBuilder
 
             logger.debug("Starting build of view({}.{}). Flushing base table {}.{}",
                          ksName, view.name, ksName, baseCfs.name);
-            baseCfs.forceBlockingFlush();
+            baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VIEW_BUILD_STARTED);
 
             loadStatusAndBuild();
         }

src/java/org/apache/cassandra/index/Index.java

@@ -26,6 +26,7 @@ import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.function.BiFunction;
 
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.*;

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
-import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.*;
@@ -51,6 +50,7 @@ import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.*;
@@ -65,8 +65,6 @@ import org.apache.cassandra.notifications.SSTableAddedNotification;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.pager.SinglePartitionPager;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.ProtocolVersion;
@@ -261,7 +259,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      * Adds and builds a index
      *
      * @param indexDef the IndexMetadata describing the index
-     * @param isNewCF true if the index is added as part of a new table/columnfamily (i.e. loading a CF at startup), 
+     * @param isNewCF true if the index is added as part of a new table/columnfamily (i.e. loading a CF at startup),
      * false for all other cases (i.e. newly added index)
      */
     public synchronized Future<?> addIndex(IndexMetadata indexDef, boolean isNewCF)
@@ -378,7 +376,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
         // Once we are tracking new writes, flush any memtable contents to not miss them from the sstable-based rebuild
         if (needsFlush)
-            baseCfs.forceBlockingFlush();
+            baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
 
         // Now that we are tracking new writes and we haven't left untracked contents on the memtables, we are ready to
         // index the sstables
@@ -618,7 +616,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      *
      * @param indexes the index to be marked as building
      * @param isFullRebuild {@code true} if this method is invoked as a full index rebuild, {@code false} otherwise
-     * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup), 
+     * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup),
      * {@code false} for all other cases (i.e. newly added index)
      */
     private synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
@@ -669,7 +667,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             if (writableIndexes.put(indexName, index) == null)
                 logger.info("Index [{}] became writable after successful build.", indexName);
         }
-        
+
         AtomicInteger counter = inProgressBuilds.get(indexName);
         if (counter != null)
         {
@@ -835,7 +833,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         {
             indexes.forEach(index ->
                             index.getBackingTable()
-                                 .map(cfs -> wait.add(cfs.forceFlush()))
+                                 .map(cfs -> wait.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_COMPLETED)))
                                  .orElseGet(() -> nonCfsIndexes.add(index)));
         }
 
@@ -892,7 +890,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
     /**
      * When building an index against existing data in sstables, add the given partition to the index
-     * 
+     *
      * @param key the key for the partition being indexed
      * @param indexes the indexes that must be updated
      * @param pageSize the number of {@link Unfiltered} objects to process in a single page
@@ -905,12 +903,12 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
         if (!indexes.isEmpty())
         {
-            SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(), 
-                                                                               FBUtilities.nowInSeconds(), 
-                                                                               ColumnFilter.selection(columns), 
-                                                                               RowFilter.NONE, 
-                                                                               DataLimits.NONE, 
-                                                                               key, 
+            SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(),
+                                                                               FBUtilities.nowInSeconds(),
+                                                                               ColumnFilter.selection(columns),
+                                                                               RowFilter.NONE,
+                                                                               DataLimits.NONE,
+                                                                               key,
                                                                                new ClusteringIndexSliceFilter(Slices.ALL, false));
 
             int nowInSec = cmd.nowInSec();
@@ -1201,7 +1199,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     {
         if (!hasIndexes())
             return UpdateTransaction.NO_OP;
-        
+
         ArrayList<Index.Indexer> idxrs = new ArrayList<>();
         for (Index i : writableIndexes.values())
         {
@@ -1209,7 +1207,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             if (idxr != null)
                 idxrs.add(idxr);
         }
-        
+
         if (idxrs.size() == 0)
             return UpdateTransaction.NO_OP;
         else

src/java/org/apache/cassandra/index/internal/CassandraIndex.java

@@ -184,7 +184,7 @@ public abstract class CassandraIndex implements Index
     public Callable<Void> getBlockingFlushTask()
     {
         return () -> {
-            indexCfs.forceBlockingFlush();
+            indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_TABLE_FLUSH);
             return null;
         };
     }
@@ -659,7 +659,7 @@ public abstract class CassandraIndex implements Index
         CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true);
         CompactionManager.instance.waitForCessation(cfss, (sstable) -> true);
         Keyspace.writeOrder.awaitNewBarrier();
-        indexCfs.forceBlockingFlush();
+        indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_REMOVED);
         indexCfs.readOrdering.awaitNewBarrier();
         indexCfs.invalidate();
     }
@@ -685,7 +685,7 @@ public abstract class CassandraIndex implements Index
     @SuppressWarnings("resource")
     private void buildBlocking()
     {
-        baseCfs.forceBlockingFlush();
+        baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
 
         try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
              Refs<SSTableReader> sstables = viewFragment.refs)
@@ -709,7 +709,7 @@ public abstract class CassandraIndex implements Index
                                                                          ImmutableSet.copyOf(sstables));
             Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
             FBUtilities.waitOnFuture(future);
-            indexCfs.forceBlockingFlush();
+            indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_COMPLETED);
         }
         logger.info("Index build of {} complete", metadata.name);
     }

src/java/org/apache/cassandra/index/sasi/SASIIndex.java

@@ -291,7 +291,7 @@ public class SASIIndex implements Index, INotificationConsumer
 
             public void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup)
             {
-                baseCfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().allocate(additionalSpace, opGroup);
+                baseCfs.getTracker().getView().getCurrentMemtable().markExtraOnHeapUsed(additionalSpace, opGroup);
             }
         };
     }

src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java

@@ -31,7 +31,7 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.UTF8Type;

src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java

@@ -34,6 +34,7 @@ import com.google.common.primitives.Longs;
 import com.google.common.util.concurrent.RateLimiter;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.rows.UnfilteredSource;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -138,7 +139,7 @@ import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQu
  *
  * TODO: fill in details about Tracker and lifecycle interactions for tools, and for compaction strategies
  */
-public abstract class SSTableReader extends SSTable implements SelfRefCounted<SSTableReader>
+public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted<SSTableReader>
 {
     private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
 
@@ -1419,13 +1420,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
                                                  boolean permitMatchPastLast,
                                                  SSTableReadsListener listener);
 
-    public abstract UnfilteredRowIterator iterator(DecoratedKey key,
-                                                   Slices slices,
-                                                   ColumnFilter selectedColumns,
-                                                   boolean reversed,
-                                                   SSTableReadsListener listener);
-
-    public abstract UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed);
+    public abstract UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed);
 
     public abstract UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly);
 
@@ -1586,14 +1581,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
      */
     public abstract ISSTableScanner getScanner(Iterator<AbstractBounds<PartitionPosition>> rangeIterator);
 
-    /**
-     * @param columns the columns to return.
-     * @param dataRange filter to use when reading the columns
-     * @param listener a listener used to handle internal read events
-     * @return A Scanner for seeking over the rows of the SSTable.
-     */
-    public abstract ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener);
-
     public FileDataInput getFileDataInput(long position)
     {
         return dfile.createReader(position);

src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java

@@ -382,8 +382,21 @@ public abstract class SSTableWriter extends SSTable implements Transactional
         FileUtils.createHardLinkWithoutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
     }
 
+    /**
+     * Parameters for calculating the expected size of an sstable. Exposed on memtable flush sets (i.e. collected
+     * subsets of a memtable that will be written to sstables).
+     */
+    public interface SSTableSizeParameters
+    {
+        long partitionCount();
+        long partitionKeysSize();
+        long dataSize();
+    }
+
     public static abstract class Factory
     {
+        public abstract long estimateSize(SSTableSizeParameters parameters);
+
         public abstract SSTableWriter open(Descriptor descriptor,
                                            long keyCount,
                                            long repairedAt,

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -78,6 +78,15 @@ public class BigFormat implements SSTableFormat
 
     static class WriterFactory extends SSTableWriter.Factory
     {
+        @Override
+        public long estimateSize(SSTableWriter.SSTableSizeParameters parameters)
+        {
+            return (long) ((parameters.partitionKeysSize() // index entries
+                            + parameters.partitionKeysSize() // keys in data file
+                            + parameters.dataSize()) // data
+                           * 1.2); // bloom filter and row index overhead
+        }
+
         @Override
         public SSTableWriter open(Descriptor descriptor,
                                   long keyCount,

src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java

@@ -57,18 +57,18 @@ public class BigTableReader extends SSTableReader
         super(builder);
     }
 
-    public UnfilteredRowIterator iterator(DecoratedKey key,
-                                          Slices slices,
-                                          ColumnFilter selectedColumns,
-                                          boolean reversed,
-                                          SSTableReadsListener listener)
+    public UnfilteredRowIterator rowIterator(DecoratedKey key,
+                                             Slices slices,
+                                             ColumnFilter selectedColumns,
+                                             boolean reversed,
+                                             SSTableReadsListener listener)
     {
         RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ, listener);
-        return iterator(null, key, rie, slices, selectedColumns, reversed);
+        return rowIterator(null, key, rie, slices, selectedColumns, reversed);
     }
 
     @SuppressWarnings("resource")
-    public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
+    public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
     {
         if (indexEntry == null)
             return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed);
@@ -78,7 +78,7 @@ public class BigTableReader extends SSTableReader
     }
 
     @Override
-    public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
+    public ISSTableScanner partitionIterator(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
     {
         return BigTableScanner.getScanner(this, columns, dataRange, listener);
     }

src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java

@@ -365,7 +365,7 @@ public class BigTableScanner implements ISSTableScanner
                             }
 
                             ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey());
-                            return sstable.iterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed());
+                            return sstable.rowIterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed());
                         }
                         catch (CorruptSSTableException | IOException e)
                         {
@@ -441,5 +441,10 @@ public class BigTableScanner implements ISSTableScanner
         {
             return null;
         }
+
+        public int getMinLocalDeletionTime()
+        {
+            return DeletionTime.LIVE.localDeletionTime();
+        }
     }
 }

src/java/org/apache/cassandra/metrics/TableMetrics.java

@@ -38,7 +38,7 @@ import org.apache.commons.lang3.ArrayUtils;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.index.SecondaryIndexManager;
@@ -392,11 +392,16 @@ public class TableMetrics
      *
      * @param cfs ColumnFamilyStore to measure metrics
      */
-    public TableMetrics(final ColumnFamilyStore cfs)
+    public TableMetrics(final ColumnFamilyStore cfs, ReleasableMetric memtableMetrics)
     {
         factory = new TableMetricNameFactory(cfs, "Table");
         aliasFactory = new TableMetricNameFactory(cfs, "ColumnFamily");
 
+        if (memtableMetrics != null)
+        {
+            all.add(memtableMetrics);
+        }
+
         samplers = new EnumMap<>(SamplerType.class);
         topReadPartitionFrequency = new FrequencySampler<ByteBuffer>()
         {
@@ -441,16 +446,16 @@ public class TableMetrics
         samplers.put(SamplerType.LOCAL_READ_TIME, topLocalReadQueryTime);
 
         memtableColumnsCount = createTableGauge("MemtableColumnsCount", 
-                                                () -> cfs.getTracker().getView().getCurrentMemtable().getOperations());
+                                                () -> cfs.getTracker().getView().getCurrentMemtable().operationCount());
 
         // MemtableOnHeapSize naming deprecated in 4.0
         memtableOnHeapDataSize = createTableGaugeWithDeprecation("MemtableOnHeapDataSize", "MemtableOnHeapSize", 
-                                                                 () -> cfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns(), 
+                                                                 () -> Memtable.getMemoryUsage(cfs.getTracker().getView().getCurrentMemtable()).ownsOnHeap,
                                                                  new GlobalTableGauge("MemtableOnHeapDataSize"));
 
         // MemtableOffHeapSize naming deprecated in 4.0
         memtableOffHeapDataSize = createTableGaugeWithDeprecation("MemtableOffHeapDataSize", "MemtableOffHeapSize", 
-                                                                  () -> cfs.getTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns(), 
+                                                                  () -> Memtable.getMemoryUsage(cfs.getTracker().getView().getCurrentMemtable()).ownsOffHeap,
                                                                   new GlobalTableGauge("MemtableOnHeapDataSize"));
         
         memtableLiveDataSize = createTableGauge("MemtableLiveDataSize", 
@@ -461,10 +466,7 @@ public class TableMetrics
         {
             public Long getValue()
             {
-                long size = 0;
-                for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes())
-                    size += cfs2.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns();
-                return size;
+                return getMemoryUsageWithIndexes(cfs).ownsOnHeap;
             }
         }, new GlobalTableGauge("AllMemtablesOnHeapDataSize"));
 
@@ -473,10 +475,7 @@ public class TableMetrics
         {
             public Long getValue()
             {
-                long size = 0;
-                for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes())
-                    size += cfs2.getTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns();
-                return size;
+                return getMemoryUsageWithIndexes(cfs).ownsOffHeap;
             }
         }, new GlobalTableGauge("AllMemtablesOffHeapDataSize"));
         allMemtablesLiveDataSize = createTableGauge("AllMemtablesLiveDataSize", new Gauge<Long>()
@@ -946,6 +945,15 @@ public class TableMetrics
         rowIndexSize = createTableHistogram("RowIndexSize", cfs.keyspace.metric.rowIndexSize, false);
     }
 
+    private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs)
+    {
+        Memtable.MemoryUsage usage = Memtable.newMemoryUsage();
+        cfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
+        for (ColumnFamilyStore indexCfs : cfs.indexManager.getAllIndexColumnFamilyStores())
+            indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
+        return usage;
+    }
+
     public void updateSSTableIterated(int count)
     {
         sstablesPerReadHistogram.update(count);

src/java/org/apache/cassandra/notifications/MemtableDiscardedNotification.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.notifications;
 
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 
 public class MemtableDiscardedNotification implements INotification
 {

src/java/org/apache/cassandra/notifications/MemtableRenewedNotification.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.notifications;
 
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 
 public class MemtableRenewedNotification implements INotification
 {

src/java/org/apache/cassandra/notifications/MemtableSwitchedNotification.java

@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.notifications;
 
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 
 public class MemtableSwitchedNotification implements INotification
 {

src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java

@@ -21,7 +21,7 @@ import java.util.Optional;
 
 import javax.annotation.Nullable;
 
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 
 /**

src/java/org/apache/cassandra/repair/consistent/LocalSessions.java

@@ -604,7 +604,7 @@ public class LocalSessions
     {
         TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
         ColumnFamilyStore cfm = Schema.instance.getColumnFamilyStoreInstance(tid);
-        cfm.forceBlockingFlush();
+        cfm.forceBlockingFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED);
     }
 
     /**

src/java/org/apache/cassandra/schema/MemtableParams.java

@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.schema;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.InheritingClass;
+import org.apache.cassandra.config.ParameterizedClass;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.memtable.SkipListMemtableFactory;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+/**
+ * Memtable types and options are specified with these parameters. Memtable classes must either contain a static
+ * {@code FACTORY} field (if they take no arguments other than class), or implement a
+ * {@code factory(Map<String, String>)} method.
+ *
+ * The latter should consume any further options (using {@code map.remove}).
+ *
+ * See Memtable_API.md for further details on the configuration and usage of memtable implementations.
+  */
+public final class MemtableParams
+{
+    private final Memtable.Factory factory;
+    private final String configurationKey;
+
+    private MemtableParams(Memtable.Factory factory, String configurationKey)
+    {
+        this.configurationKey = configurationKey;
+        this.factory = factory;
+    }
+
+    public String configurationKey()
+    {
+        return configurationKey;
+    }
+
+    public Memtable.Factory factory()
+    {
+        return factory;
+    }
+
+    @Override
+    public String toString()
+    {
+        return configurationKey;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof MemtableParams))
+            return false;
+
+        MemtableParams c = (MemtableParams) o;
+
+        return Objects.equal(configurationKey, c.configurationKey);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return configurationKey.hashCode();
+    }
+
+    private static final String DEFAULT_CONFIGURATION_KEY = "default";
+    private static final Memtable.Factory DEFAULT_MEMTABLE_FACTORY = SkipListMemtableFactory.INSTANCE;
+    private static final ParameterizedClass DEFAULT_CONFIGURATION = SkipListMemtableFactory.CONFIGURATION;
+    private static final Map<String, ParameterizedClass>
+        CONFIGURATION_DEFINITIONS = expandDefinitions(DatabaseDescriptor.getMemtableConfigurations());
+    private static final Map<String, MemtableParams> CONFIGURATIONS = new HashMap<>();
+    public static final MemtableParams DEFAULT = get(null);
+
+    public static MemtableParams get(String key)
+    {
+        if (key == null)
+            key = DEFAULT_CONFIGURATION_KEY;
+
+        synchronized (CONFIGURATIONS)
+        {
+            return CONFIGURATIONS.computeIfAbsent(key, MemtableParams::parseConfiguration);
+        }
+    }
+
+    @VisibleForTesting
+    static Map<String, ParameterizedClass> expandDefinitions(Map<String, InheritingClass> memtableConfigurations)
+    {
+        if (memtableConfigurations == null)
+            return ImmutableMap.of(DEFAULT_CONFIGURATION_KEY, DEFAULT_CONFIGURATION);
+
+        LinkedHashMap<String, ParameterizedClass> configs = new LinkedHashMap<>(memtableConfigurations.size() + 1);
+
+        // If default is not overridden, add an entry first so that other configurations can inherit from it.
+        // If it is, process it in its point of definition, so that the default can inherit from another configuration.
+        if (!memtableConfigurations.containsKey(DEFAULT_CONFIGURATION_KEY))
+            configs.put(DEFAULT_CONFIGURATION_KEY, DEFAULT_CONFIGURATION);
+
+        for (Map.Entry<String, InheritingClass> en : memtableConfigurations.entrySet())
+            configs.put(en.getKey(), en.getValue().resolve(configs));
+
+        return ImmutableMap.copyOf(configs);
+    }
+
+    private static MemtableParams parseConfiguration(String configurationKey)
+    {
+        ParameterizedClass definition = CONFIGURATION_DEFINITIONS.get(configurationKey);
+
+        if (definition == null)
+            throw new ConfigurationException("Memtable configuration \"" + configurationKey + "\" not found.");
+        return new MemtableParams(getMemtableFactory(definition), configurationKey);
+    }
+
+
+    private static Memtable.Factory getMemtableFactory(ParameterizedClass options)
+    {
+        // Special-case this so that we don't initialize memtable class for tests that need to delay that.
+        if (options == DEFAULT_CONFIGURATION)
+            return DEFAULT_MEMTABLE_FACTORY;
+
+        String className = options.class_name;
+        if (className == null || className.isEmpty())
+            throw new ConfigurationException("The 'class_name' option must be specified.");
+
+        className = className.contains(".") ? className : "org.apache.cassandra.db.memtable." + className;
+        try
+        {
+            Memtable.Factory factory;
+            Class<?> clazz = Class.forName(className);
+            final Map<String, String> parametersCopy = options.parameters != null
+                                                       ? new HashMap<>(options.parameters)
+                                                       : new HashMap<>();
+            try
+            {
+                Method factoryMethod = clazz.getDeclaredMethod("factory", Map.class);
+                factory = (Memtable.Factory) factoryMethod.invoke(null, parametersCopy);
+            }
+            catch (NoSuchMethodException e)
+            {
+                // continue with FACTORY field
+                Field factoryField = clazz.getDeclaredField("FACTORY");
+                factory = (Memtable.Factory) factoryField.get(null);
+            }
+            if (!parametersCopy.isEmpty())
+                throw new ConfigurationException("Memtable class " + className + " does not accept any futher parameters, but " +
+                                                 parametersCopy + " were given.");
+            return factory;
+        }
+        catch (NoSuchFieldException | ClassNotFoundException | IllegalAccessException | InvocationTargetException | ClassCastException e)
+        {
+            if (e.getCause() instanceof ConfigurationException)
+                throw (ConfigurationException) e.getCause();
+            throw new ConfigurationException("Could not create memtable factory for class " + options, e);
+        }
+    }
+}

src/java/org/apache/cassandra/schema/SchemaEvent.java

@@ -219,6 +219,7 @@ public final class SchemaEvent extends DiagnosticEvent
         ret.put("caching", repr(params.caching));
         ret.put("compaction", repr(params.compaction));
         ret.put("compression", repr(params.compression));
+        ret.put("memtable", repr(params.memtable));
         if (params.speculativeRetry != null) ret.put("speculativeRetry", params.speculativeRetry.kind().name());
         return ret;
     }
@@ -247,6 +248,11 @@ public final class SchemaEvent extends DiagnosticEvent
         return ret;
     }
 
+    private String repr(MemtableParams params)
+    {
+        return params.configurationKey();
+    }
+
     private HashMap<String, Serializable> repr(IndexMetadata index)
     {
         HashMap<String, Serializable> ret = new HashMap<>();

src/java/org/apache/cassandra/schema/SchemaKeyspace.java

@@ -105,6 +105,7 @@ public final class SchemaKeyspace
               + "comment text,"
               + "compaction frozen<map<text, text>>,"
               + "compression frozen<map<text, text>>,"
+              + "memtable text,"
               + "crc_check_chance double,"
               + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake
               + "default_time_to_live int,"
@@ -172,6 +173,7 @@ public final class SchemaKeyspace
               + "comment text,"
               + "compaction frozen<map<text, text>>,"
               + "compression frozen<map<text, text>>,"
+              + "memtable text,"
               + "crc_check_chance double,"
               + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake
               + "default_time_to_live int,"
@@ -327,7 +329,7 @@ public final class SchemaKeyspace
     private static void flush()
     {
         if (!DatabaseDescriptor.isUnsafeSystem())
-            ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush()));
+            ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED)));
     }
 
     /**
@@ -400,7 +402,7 @@ public final class SchemaKeyspace
 
                     DecoratedKey key = partition.partitionKey();
                     Mutation.PartitionUpdateCollector puCollector = mutationMap.computeIfAbsent(key, k -> new Mutation.PartitionUpdateCollector(SchemaConstants.SCHEMA_KEYSPACE_NAME, key));
-                    puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter()));
+                    puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter()).withOnlyPresentColumns());
                 }
             }
         }
@@ -510,6 +512,7 @@ public final class SchemaKeyspace
     {
         Row.SimpleBuilder rowBuilder = builder.update(Tables)
                                               .row(table.name)
+                                              .deletePrevious()
                                               .add("id", table.id.asUUID())
                                               .add("flags", TableMetadata.Flag.toStringSet(table.flags));
 
@@ -555,6 +558,11 @@ public final class SchemaKeyspace
         // node sends table schema to a < 3.8 versioned node with an unknown column.
         if (DatabaseDescriptor.isCDCEnabled())
             builder.add("cdc", params.cdc);
+
+        // As above, only add the memtable column if the table uses a non-default memtable configuration to avoid RTE
+        // in mixed operation with pre-4.1 versioned node during upgrades.
+        if (params.memtable != MemtableParams.DEFAULT)
+            builder.add("memtable", params.memtable.configurationKey());
     }
 
     private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder)
@@ -714,6 +722,7 @@ public final class SchemaKeyspace
         TableMetadata table = view.metadata;
         Row.SimpleBuilder rowBuilder = builder.update(Views)
                                               .row(view.name())
+                                              .deletePrevious()
                                               .add("include_all_columns", view.includeAllColumns)
                                               .add("base_table_id", view.baseTableId.asUUID())
                                               .add("base_table_name", view.baseTableName)
@@ -951,6 +960,7 @@ public final class SchemaKeyspace
                           .comment(row.getString("comment"))
                           .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction")))
                           .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression")))
+                          .memtable(MemtableParams.get(row.has("memtable") ? row.getString("memtable") : null)) // memtable column was introduced in 4.1
                           .defaultTimeToLive(row.getInt("default_time_to_live"))
                           .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance))
                           .gcGraceSeconds(row.getInt("gc_grace_seconds"))

src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java

@@ -43,6 +43,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.Range;
@@ -368,7 +369,7 @@ public final class SystemDistributedKeyspace
     {
         String buildReq = "DELETE FROM %s.%s WHERE keyspace_name = ? AND view_name = ?";
         QueryProcessor.executeInternal(format(buildReq, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS), keyspaceName, viewName);
-        forceBlockingFlush(VIEW_BUILD_STATUS);
+        forceBlockingFlush(VIEW_BUILD_STATUS, ColumnFamilyStore.FlushReason.INTERNALLY_FORCED);
     }
 
     private static void processSilent(String fmtQry, String... values)
@@ -388,10 +389,12 @@ public final class SystemDistributedKeyspace
         }
     }
 
-    public static void forceBlockingFlush(String table)
+    public static void forceBlockingFlush(String table, ColumnFamilyStore.FlushReason reason)
     {
         if (!DatabaseDescriptor.isUnsafeSystem())
-            FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(table).forceFlush());
+            FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME)
+                                             .getColumnFamilyStore(table)
+                                             .forceFlush(reason));
     }
 
     private enum RepairState

src/java/org/apache/cassandra/schema/TableMetadata.java

@@ -867,6 +867,13 @@ public class TableMetadata implements SchemaElement
             return this;
         }
 
+        public Builder memtable(MemtableParams val)
+        {
+            params.memtable(val);
+            return this;
+        }
+
+
         public Builder isCounter(boolean val)
         {
             return flag(Flag.COUNTER, val);

src/java/org/apache/cassandra/schema/TableParams.java

@@ -46,6 +46,7 @@ public final class TableParams
         COMMENT,
         COMPACTION,
         COMPRESSION,
+        MEMTABLE,
         DEFAULT_TIME_TO_LIVE,
         EXTENSIONS,
         GC_GRACE_SECONDS,
@@ -78,6 +79,7 @@ public final class TableParams
     public final CachingParams caching;
     public final CompactionParams compaction;
     public final CompressionParams compression;
+    public final MemtableParams memtable;
     public final ImmutableMap<String, ByteBuffer> extensions;
     public final boolean cdc;
     public final ReadRepairStrategy readRepair;
@@ -99,6 +101,7 @@ public final class TableParams
         caching = builder.caching;
         compaction = builder.compaction;
         compression = builder.compression;
+        memtable = builder.memtable;
         extensions = builder.extensions;
         cdc = builder.cdc;
         readRepair = builder.readRepair;
@@ -116,6 +119,7 @@ public final class TableParams
                             .comment(params.comment)
                             .compaction(params.compaction)
                             .compression(params.compression)
+                            .memtable(params.memtable)
                             .crcCheckChance(params.crcCheckChance)
                             .defaultTimeToLive(params.defaultTimeToLive)
                             .gcGraceSeconds(params.gcGraceSeconds)
@@ -178,6 +182,9 @@ public final class TableParams
 
         if (memtableFlushPeriodInMs < 0)
             fail("%s must be greater than or equal to 0 (got %s)", Option.MEMTABLE_FLUSH_PERIOD_IN_MS, memtableFlushPeriodInMs);
+
+        if (cdc && memtable.factory().writesShouldSkipCommitLog())
+            fail("CDC cannot work if writes skip the commit log. Check your memtable configuration.");
     }
 
     private static void fail(String format, Object... args)
@@ -208,6 +215,7 @@ public final class TableParams
             && caching.equals(p.caching)
             && compaction.equals(p.compaction)
             && compression.equals(p.compression)
+            && memtable.equals(p.memtable)
             && extensions.equals(p.extensions)
             && cdc == p.cdc
             && readRepair == p.readRepair;
@@ -228,6 +236,7 @@ public final class TableParams
                                 caching,
                                 compaction,
                                 compression,
+                                memtable,
                                 extensions,
                                 cdc,
                                 readRepair);
@@ -249,6 +258,7 @@ public final class TableParams
                           .add(Option.CACHING.toString(), caching)
                           .add(Option.COMPACTION.toString(), compaction)
                           .add(Option.COMPRESSION.toString(), compression)
+                          .add(Option.MEMTABLE.toString(), memtable)
                           .add(Option.EXTENSIONS.toString(), extensions)
                           .add(Option.CDC.toString(), cdc)
                           .add(Option.READ_REPAIR.toString(), readRepair)
@@ -272,6 +282,8 @@ public final class TableParams
                .newLine()
                .append("AND compression = ").append(compression.asMap())
                .newLine()
+               .append("AND memtable = ").appendWithSingleQuotes(memtable.configurationKey())
+               .newLine()
                .append("AND crc_check_chance = ").append(crcCheckChance)
                .newLine();
 
@@ -315,6 +327,7 @@ public final class TableParams
         private CachingParams caching = CachingParams.DEFAULT;
         private CompactionParams compaction = CompactionParams.DEFAULT;
         private CompressionParams compression = CompressionParams.DEFAULT;
+        private MemtableParams memtable = MemtableParams.DEFAULT;
         private ImmutableMap<String, ByteBuffer> extensions = ImmutableMap.of();
         private boolean cdc;
         private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING;
@@ -400,6 +413,12 @@ public final class TableParams
             return this;
         }
 
+        public Builder memtable(MemtableParams val)
+        {
+            memtable = val;
+            return this;
+        }
+
         public Builder compression(CompressionParams val)
         {
             compression = val;

src/java/org/apache/cassandra/service/StorageService.java

@@ -355,6 +355,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         setGossipTokens(localTokens);
         tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort());
         setMode(Mode.NORMAL, false);
+        invalidateLocalRanges();
     }
 
     public void setGossipTokens(Collection<Token> tokens)
@@ -1866,7 +1867,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
 
         // Force disk boundary invalidation now that local tokens are set
-        invalidateDiskBoundaries();
+        invalidateLocalRanges();
         repairPaxosForTopologyChange("bootstrap");
 
         Future<StreamState> bootstrapStream = startBootstrap(tokens);
@@ -1900,7 +1901,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update
     }
 
-    private void invalidateDiskBoundaries()
+    private void invalidateLocalRanges()
     {
         for (Keyspace keyspace : Keyspace.all())
         {
@@ -1908,7 +1909,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             {
                 for (final ColumnFamilyStore store : cfs.concatWithIndexes())
                 {
-                    store.invalidateDiskBoundaries();
+                    store.invalidateLocalRanges();
                 }
             }
         }
@@ -2834,6 +2835,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
         if (!tokensToUpdateInSystemKeyspace.isEmpty())
             SystemKeyspace.updateTokens(endpoint, tokensToUpdateInSystemKeyspace);
+
+        // Tokens changed, the local range ownership probably changed too.
+        invalidateLocalRanges();
     }
 
     @VisibleForTesting
@@ -2954,6 +2958,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         if (isMoving || operationMode == Mode.MOVING)
         {
             tokenMetadata.removeFromMoving(endpoint);
+            // The above may change the local ownership.
+            invalidateLocalRanges();
             notifyMoved(endpoint);
         }
         else if (!isMember) // prior to this, the node was not a member
@@ -4219,7 +4225,21 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tableNames))
         {
             logger.debug("Forcing flush on keyspace {}, CF {}", keyspaceName, cfStore.name);
-            cfStore.forceBlockingFlush();
+            cfStore.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        }
+    }
+
+    /**
+     * Flush all memtables for a keyspace and column families.
+     * @param keyspaceName
+     * @throws IOException
+     */
+    public void forceKeyspaceFlush(String keyspaceName, ColumnFamilyStore.FlushReason reason) throws IOException
+    {
+        for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName))
+        {
+            logger.debug("Forcing flush on keyspace {}, CF {}", keyspaceName, cfStore.name);
+            cfStore.forceBlockingFlush(reason);
         }
     }
 
@@ -5282,7 +5302,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             for (Keyspace keyspace : Keyspace.nonSystem())
             {
                 for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
-                    flushes.add(cfs.forceFlush());
+                    flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.DRAIN));
             }
             // wait for the flushes.
             // TODO this is a godawful way to track progress, since they flush in parallel.  a long one could
@@ -5314,7 +5334,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             for (Keyspace keyspace : Keyspace.system())
             {
                 for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
-                    flushes.add(cfs.forceFlush());
+                    flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.DRAIN));
             }
             FBUtilities.waitOnFutures(flushes);
 

src/java/org/apache/cassandra/service/disk/usage/DiskUsageMonitor.java

@@ -41,9 +41,9 @@ import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DataStorageSpec;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.Memtable;
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.db.guardrails.GuardrailsConfig;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.io.util.FileUtils;
 
 /**

src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java

@@ -34,6 +34,7 @@ import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.dht.Range;
@@ -42,6 +43,7 @@ import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
@@ -144,7 +146,7 @@ public class PaxosUncommittedIndex implements Index, PaxosUncommittedTracker.Upd
         for (int j=0, jsize=dataRanges.size(); j<jsize; j++)
         {
             for (int i=0, isize=memtables.size(); i<isize; i++)
-                iters.add(memtables.get(i).makePartitionIterator(memtableColumnFilter, dataRanges.get(j)));
+                iters.add(memtables.get(i).partitionIterator(memtableColumnFilter, dataRanges.get(j), SSTableReadsListener.NOOP_LISTENER));
         }
 
         return getPaxosUpdates(iters, tableId, false);
@@ -152,7 +154,7 @@ public class PaxosUncommittedIndex implements Index, PaxosUncommittedTracker.Upd
 
     public CloseableIterator<PaxosKeyState> flushIterator(Memtable flushing)
     {
-        List<UnfilteredPartitionIterator> iters = singletonList(flushing.makePartitionIterator(memtableColumnFilter, FULL_RANGE));
+        List<UnfilteredPartitionIterator> iters = singletonList(flushing.partitionIterator(memtableColumnFilter, FULL_RANGE, SSTableReadsListener.NOOP_LISTENER));
         return getPaxosUpdates(iters, null, true);
     }
 

src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java

@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.util.File;

src/java/org/apache/cassandra/streaming/StreamSession.java

@@ -993,7 +993,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
     {
         List<Future<?>> flushes = new ArrayList<>();
         for (ColumnFamilyStore cfs : stores)
-            flushes.add(cfs.forceFlush());
+            flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.STREAMING));
         FBUtilities.waitOnFutures(flushes);
     }
 

src/java/org/apache/cassandra/utils/memory/HeapPool.java

@@ -20,7 +20,6 @@ package org.apache.cassandra.utils.memory;
 
 import java.nio.ByteBuffer;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.utils.Shared;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
@@ -35,7 +34,7 @@ public class HeapPool extends MemtablePool
         super(maxOnHeapMemory, 0, cleanupThreshold, cleaner);
     }
 
-    public MemtableAllocator newAllocator(ColumnFamilyStore table)
+    public MemtableAllocator newAllocator(String table)
     {
         return new Allocator(this);
     }
@@ -94,9 +93,9 @@ public class HeapPool extends MemtablePool
             super(maxOnHeapMemory, 0, cleanupThreshold, cleaner);
         }
 
-        public MemtableAllocator newAllocator(ColumnFamilyStore cfs)
+        public MemtableAllocator newAllocator(String table)
         {
-            return new Allocator(this, cfs == null ? "" : cfs.keyspace.getName() + '.' + cfs.name);
+            return new Allocator(this, table);
         }
 
         private static class Allocator extends MemtableBufferAllocator

src/java/org/apache/cassandra/utils/memory/MemtablePool.java

@@ -27,7 +27,6 @@ import com.google.common.base.Preconditions;
 
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Timer;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.metrics.CassandraMetricsRegistry;
 import org.apache.cassandra.metrics.DefaultNameFactory;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
@@ -82,7 +81,7 @@ public abstract class MemtablePool
         ExecutorUtils.shutdownNowAndWait(timeout, unit, cleaner);
     }
 
-    public abstract MemtableAllocator newAllocator(ColumnFamilyStore table);
+    public abstract MemtableAllocator newAllocator(String table);
 
     public boolean needsCleaning()
     {

src/java/org/apache/cassandra/utils/memory/NativePool.java

@@ -18,8 +18,6 @@
  */
 package org.apache.cassandra.utils.memory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-
 public class NativePool extends MemtablePool
 {
     public NativePool(long maxOnHeapMemory, long maxOffHeapMemory, float cleanThreshold, MemtableCleaner cleaner)
@@ -28,7 +26,7 @@ public class NativePool extends MemtablePool
     }
 
     @Override
-    public NativeAllocator newAllocator(ColumnFamilyStore table)
+    public NativeAllocator newAllocator(String table)
     {
         return new NativeAllocator(this);
     }

src/java/org/apache/cassandra/utils/memory/SlabPool.java

@@ -18,8 +18,6 @@
  */
 package org.apache.cassandra.utils.memory;
 
-import org.apache.cassandra.db.ColumnFamilyStore;
-
 public class SlabPool extends MemtablePool
 {
     final boolean allocateOnHeap;
@@ -30,7 +28,7 @@ public class SlabPool extends MemtablePool
         this.allocateOnHeap = maxOffHeapMemory == 0;
     }
 
-    public MemtableAllocator newAllocator(ColumnFamilyStore table)
+    public MemtableAllocator newAllocator(String table)
     {
         return new SlabAllocator(onHeap.newAllocator(), offHeap.newAllocator(), allocateOnHeap);
     }

test/conf/cassandra.yaml

@@ -61,3 +61,45 @@ local_read_size_warn_threshold: 4096KiB
 local_read_size_fail_threshold: 8192KiB
 row_index_read_size_warn_threshold: 4096KiB
 row_index_read_size_fail_threshold: 8192KiB
+
+memtable:
+    configurations:
+        skiplist:
+            inherits: default
+            class_name: SkipListMemtable
+        skiplist_sharded:
+            class_name: ShardedSkipListMemtable
+            parameters:
+                serialize_writes: false
+                shards: 4
+        skiplist_sharded_locking:
+            inherits: skiplist_sharded
+            parameters:
+                serialize_writes: true
+        skiplist_remapped:
+            inherits: skiplist
+        test_fullname:
+            inherits: default
+            class_name: org.apache.cassandra.db.memtable.TestMemtable
+        test_shortname:
+            class_name: TestMemtable
+            parameters:
+                skiplist: true  # note: YAML must interpret this as string, not a boolean
+        test_empty_class:
+            class_name: ""
+        test_missing_class:
+            parameters:
+        test_unknown_class:
+            class_name: NotExisting
+        test_invalid_param:
+            class_name: SkipListMemtable
+            parameters:
+                invalid: throw
+        test_invalid_extra_param:
+            inherits: test_shortname
+            parameters:
+                invalid: throw
+        test_invalid_factory_method:
+            class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryMethod
+        test_invalid_factory_field:
+            class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryField

test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java

@@ -27,6 +27,7 @@ import harry.core.Configuration;
 import harry.core.Run;
 import harry.ddl.SchemaSpec;
 import harry.model.clock.OffsetClock;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.distributed.Cluster;
@@ -124,7 +125,7 @@ public abstract class FuzzTestBase extends TestBaseImpl
                 if (schema.staticColumns != null)
                     writeStatic(current, 0, current, current, true).applyUnsafe();
             }
-            store.forceBlockingFlush();
+            Util.flush(store);
             return null;
         }
     }

test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java

@@ -34,6 +34,7 @@ import harry.operations.Relation;
 import harry.operations.Query;
 import harry.operations.QueryGenerator;
 import harry.util.BitSet;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.AbstractMarker;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.Operator;
@@ -113,7 +114,7 @@ public class SSTableGenerator
 
     public Collection<SSTableReader> flush()
     {
-        store.forceBlockingFlush();
+        Util.flush(store);
         sstables.removeAll(store.getLiveSSTables());
 
         Set<SSTableReader> ret = new HashSet<>(sstables);

test/distributed/org/apache/cassandra/distributed/impl/Instance.java

@@ -46,6 +46,7 @@ import javax.management.NotificationListener;
 import com.google.common.annotations.VisibleForTesting;
 
 import io.netty.util.concurrent.GlobalEventExecutor;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.auth.AuthCache;
 import org.apache.cassandra.batchlog.Batch;
 import org.apache.cassandra.batchlog.BatchlogManager;
@@ -63,12 +64,12 @@ import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Memtable;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.SystemKeyspaceMigrator41;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.compaction.CompactionLogger;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.memtable.AbstractAllocatorMemtable;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.Constants;
 import org.apache.cassandra.distributed.action.GossipHelper;
@@ -141,8 +142,8 @@ import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
 import org.apache.cassandra.utils.memory.BufferPools;
+import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
 
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@@ -513,7 +514,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
 
     public void flush(String keyspace)
     {
-        FBUtilities.waitOnFutures(Keyspace.open(keyspace).flush());
+        Util.flushKeyspace(keyspace);
     }
 
     public void forceCompact(String keyspace, String table)
@@ -768,7 +769,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                                 () -> PaxosRepair.shutdownAndWait(1L, MINUTES),
                                 () -> Ref.shutdownReferenceReaper(1L, MINUTES),
                                 () -> UncommittedTableData.shutdownAndWait(1L, MINUTES),
-                                () -> Memtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES),
+                                () -> AbstractAllocatorMemtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES),
                                 () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES),
                                 () -> SSTableReader.shutdownBlocking(1L, MINUTES),
                                 () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),

test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java

@@ -45,6 +45,7 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+import org.apache.cassandra.Util;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.Keyspace;
@@ -112,7 +113,7 @@ public class FailingRepairTest extends TestBaseImpl implements Serializable
         return () -> {
             String cfName = getCfName(type, parallelism, withTracing);
             ColumnFamilyStore cf = Keyspace.open(KEYSPACE).getColumnFamilyStore(cfName);
-            cf.forceBlockingFlush();
+            Util.flush(cf);
             Set<SSTableReader> remove = cf.getLiveSSTables();
             Set<SSTableReader> replace = new HashSet<>();
             if (type == Verb.VALIDATION_REQ)

test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java

@@ -25,6 +25,7 @@ import java.util.Set;
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.Util;
 import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
@@ -159,7 +160,7 @@ public class JVMStabilityInspectorCorruptSSTableExceptionTest extends TestBaseIm
     {
         node.runOnInstance(() -> {
             ColumnFamilyStore cf = Keyspace.open(keyspace).getColumnFamilyStore(table);
-            cf.forceBlockingFlush();
+            Util.flush(cf);
 
             Set<SSTableReader> remove = cf.getLiveSSTables();
             Set<SSTableReader> replace = new HashSet<>();
@@ -187,13 +188,13 @@ public class JVMStabilityInspectorCorruptSSTableExceptionTest extends TestBaseIm
         }
 
         @Override
-        public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+        public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
         {
             throw throwCorrupted();
         }
 
         @Override
-        public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
+        public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
         {
             throw throwCorrupted();
         }

test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java

@@ -317,7 +317,7 @@ public class PaxosRepairTest extends TestBaseImpl
                 // exception expected
             }
             Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE));
-            cluster.forEach(i -> i.runOnInstance(() -> Keyspace.open("system").getColumnFamilyStore("paxos").forceBlockingFlush()));
+            cluster.forEach(i -> i.runOnInstance(() -> Keyspace.open("system").getColumnFamilyStore("paxos").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)));
 
             CountDownLatch haveFetchedLowBound = new CountDownLatch(1);
             CountDownLatch haveReproposed = new CountDownLatch(1);
@@ -580,7 +580,7 @@ public class PaxosRepairTest extends TestBaseImpl
     private static void compactPaxos()
     {
         ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS);
-        FBUtilities.waitOnFuture(paxos.forceFlush());
+        FBUtilities.waitOnFuture(paxos.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
         FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false));
     }
 

test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java

@@ -48,11 +48,11 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.Memtable;
 import org.apache.cassandra.db.ReadQuery;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.BTreeRow;
@@ -298,7 +298,7 @@ public class PaxosRepairTest2 extends TestBaseImpl
     private static void compactPaxos()
     {
         ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS);
-        FBUtilities.waitOnFuture(paxos.forceFlush());
+        FBUtilities.waitOnFuture(paxos.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS));
         FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false));
     }
 

test/distributed/org/apache/cassandra/distributed/test/RepairTest.java

@@ -27,6 +27,7 @@ import java.util.function.Consumer;
 
 import com.google.common.collect.ImmutableMap;
 
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.distributed.shared.ClusterUtils;
 import org.apache.cassandra.utils.concurrent.Condition;
 import org.junit.AfterClass;
@@ -81,7 +82,8 @@ public class RepairTest extends TestBaseImpl
     private static void flush(ICluster<IInvokableInstance> cluster, String keyspace, int ... nodes)
     {
         for (int node : nodes)
-            cluster.get(node).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush(keyspace)));
+            cluster.get(node).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush(keyspace,
+                                                                                                     ColumnFamilyStore.FlushReason.UNIT_TESTS)));
     }
 
     private static ICluster create(Consumer<IInstanceConfig> configModifier) throws IOException

test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java

@@ -29,6 +29,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.Feature;
@@ -133,7 +134,8 @@ public class SSTableLoaderEncryptionOptionsTest extends AbstractEncryptionOption
             CLUSTER.get(1).executeInternal(String.format("INSERT INTO ssl_upload_tables.test (pk, val) VALUES (%s, '%s')",
                                                          i, Integer.toString(i)));
         }
-        CLUSTER.get(1).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush("ssl_upload_tables")));
+        CLUSTER.get(1).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush("ssl_upload_tables",
+                                                                                              ColumnFamilyStore.FlushReason.UNIT_TESTS)));
     }
 
     private static void truncateGeneratedTables() throws IOException

test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.RowIndexEntry;
 import org.apache.cassandra.db.Slices;
 import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -283,15 +284,15 @@ public abstract class ForwardingSSTableReader extends SSTableReader
     }
 
     @Override
-    public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
+    public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
     {
-        return delegate.iterator(key, slices, selectedColumns, reversed, listener);
+        return delegate.rowIterator(key, slices, selectedColumns, reversed, listener);
     }
 
     @Override
-    public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
+    public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed)
     {
-        return delegate.iterator(file, key, indexEntry, slices, selectedColumns, reversed);
+        return delegate.rowIterator(file, key, indexEntry, slices, selectedColumns, reversed);
     }
 
     @Override
@@ -385,9 +386,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader
     }
 
     @Override
-    public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
+    public UnfilteredPartitionIterator partitionIterator(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener)
     {
-        return delegate.getScanner(columns, dataRange, listener);
+        return delegate.partitionIterator(columns, dataRange, listener);
     }
 
     @Override

test/long/org/apache/cassandra/cql3/ViewLongTest.java

@@ -29,10 +29,10 @@ import org.junit.Test;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.exceptions.NoHostAvailableException;
 import com.datastax.driver.core.exceptions.WriteTimeoutException;
+import org.apache.cassandra.Util;
 import org.apache.cassandra.batchlog.BatchlogManager;
 import org.apache.cassandra.concurrent.NamedThreadFactory;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public class ViewLongTest extends ViewAbstractParameterizedTest
@@ -332,7 +332,7 @@ public class ViewLongTest extends ViewAbstractParameterizedTest
 
             updateViewWithFlush("UPDATE %s USING TTL 90  SET b = null WHERE k = 1 AND c = 2", flush);
             if (flush)
-                FBUtilities.waitOnFutures(Keyspace.open(keyspace()).flush());
+                Util.flushKeyspace(keyspace());
             assertRows(execute("select k,c,a,b from %s"));
             assertRows(executeView("select k,c,a from %s"));
 

test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java

@@ -168,7 +168,7 @@ public class LongCompactionsTest
 
                 inserted.add(key);
             }
-            cfs.forceBlockingFlush();
+            Util.flush(cfs);
             CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected);
 
             assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size());

test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java

@@ -32,6 +32,7 @@ import org.junit.Test;
 import org.apache.cassandra.config.Config;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Hex;
 
@@ -72,7 +73,7 @@ public class LongLeveledCompactionStrategyCQLTest extends CQLTester
                             throw new RuntimeException(throwable);
                         }
                     }
-                    getCurrentColumnFamilyStore().forceBlockingFlush();
+                    getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
                     Uninterruptibles.sleepUninterruptibly(r.nextInt(200), TimeUnit.MILLISECONDS);
                 }
             });

test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java

@@ -162,7 +162,7 @@ public class LongLeveledCompactionStrategyTest
         }
 
         //Flush sstable
-        store.forceBlockingFlush();
+        Util.flush(store);
 
         store.runWithCompactionsDisabled(new Callable<Void>()
         {
@@ -261,7 +261,7 @@ public class LongLeveledCompactionStrategyTest
 
             Mutation rm = new Mutation(builder.build());
             rm.apply();
-            store.forceBlockingFlush();
+            Util.flush(store);
         }
     }
 }

test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java

@@ -36,6 +36,8 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.Uninterruptibles;
+
+import org.apache.cassandra.Util;
 import org.apache.cassandra.index.internal.CollatedViewIndexBuilder;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -524,7 +526,7 @@ public class CompactionAllocationTest
                             reads.add(() -> runQuery(query, cfs.metadata.get()));
                         }
                     }
-                    cfs.forceBlockingFlush();
+                    Util.flush(cfs);
                 }
 
                 Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
@@ -639,7 +641,7 @@ public class CompactionAllocationTest
                             reads.add(() -> runQuery(query, cfs.metadata.get()));
                         }
                     }
-                    cfs.forceBlockingFlush();
+                    Util.flush(cfs);
                 }
 
                 Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
@@ -739,7 +741,7 @@ public class CompactionAllocationTest
                             reads.add(() -> runQuery(query, cfs.metadata.get()));
                         }
                     }
-                    cfs.forceBlockingFlush();
+                    Util.flush(cfs);
                 }
 
                 Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
@@ -833,7 +835,7 @@ public class CompactionAllocationTest
                                                            makeRandomString(rowWidth>>2), makeRandomString(rowWidth>>2));
                         }
                     }
-                    cfs.forceBlockingFlush();
+                    Util.flush(cfs);
                 }
 
                 for (IndexDef index : indexes)