Miscellaneous migration test fixes

Patch by Ariel Weisberg; Reviewed by David Capwell for CASSANDRA-20060

build.xml

@@ -1040,6 +1040,32 @@
       </checksum>
     </target>
 
+    <target name="bintar" depends="_artifacts-init,check"
+            description="Create Cassandra tarball and maven artifacts">
+      <tar compression="gzip" longfile="gnu"
+        destfile="${build.dir}/${final.name}-bin.tar.gz">
+
+        <!-- Everything but bin/ (default mode) -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}">
+          <include name="**"/>
+          <exclude name="bin/*" />
+          <exclude name="tools/bin/*"/>
+        </tarfileset>
+        <!-- Shell includes in bin/ (default mode) -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}">
+          <include name="bin/*.in.sh" />
+          <include name="tools/bin/*.in.sh" />
+        </tarfileset>
+        <!-- Executable scripts in bin/ -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}" mode="755">
+          <include name="bin/*"/>
+          <include name="tools/bin/*"/>
+          <exclude name="bin/*.in.sh" />
+          <exclude name="tools/bin/*.in.sh" />
+        </tarfileset>
+      </tar>
+    </target>
+
   <!-- Wrapper of build-test without dependencies, so both that target and its dependencies are skipped if the property
     no-build-test is true. This is meant to be used to run tests without actually building them, provided that they have
     been built before. All test targets depend on this, so one can run them using the no-build-test property.

conf/cassandra.yaml

@@ -2657,7 +2657,10 @@ storage_compatibility_mode: NONE
 #  journal_directory:
 #
 #  # The number of Accord shards on this node; -1 means use the number of cores
-#  shard_count: -1
+#  queue_shard_count: -1
+#
+#  # The number of Accord shards on this node; -1 means use the number of cores
+#  command_store_shard_count: -1
 #
 #  # Recover delay: the time between a transaction being initiated and a remote replica being willing to interrupt it to complete it
 #  recover_delay: 1s

modules/accord

@@ -1 +1 @@
-Subproject commit 4ec8d262a750a76744b7f6991b711f85fa41a89a
+Subproject commit 8bd9d6980350fa68a1db676a7b10940cf0541fb5

src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java

@@ -108,7 +108,6 @@ public class InfiniteLoopExecutor implements Interruptible
         };
     }
 
-
     private void loop()
     {
         boolean interrupted = false;
@@ -194,6 +193,11 @@ public class InfiniteLoopExecutor implements Interruptible
         return isTerminated();
     }
 
+    public long threadId()
+    {
+        return thread.getId();
+    }
+
     @VisibleForTesting
     public boolean isAlive()
     {

src/java/org/apache/cassandra/concurrent/Stage.java

@@ -47,7 +47,7 @@ public enum Stage
     MUTATION           (true,  "MutationStage",         "request",  DatabaseDescriptor::getConcurrentWriters,        DatabaseDescriptor::setConcurrentWriters,        Stage::multiThreadedLowSignalStage),
     COUNTER_MUTATION   (true,  "CounterMutationStage",  "request",  DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage),
     VIEW_MUTATION      (true,  "ViewMutationStage",     "request",  DatabaseDescriptor::getConcurrentViewWriters,    DatabaseDescriptor::setConcurrentViewWriters,    Stage::multiThreadedLowSignalStage),
-    ACCORD_MIGRATION   (false, "AccordMigrationStage",  "request",  DatabaseDescriptor::getConcurrentAccordOps,      DatabaseDescriptor::setConcurrentAccordOps,      Stage::multiThreadedLowSignalStage),
+    ACCORD_MIGRATION   (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage),
     GOSSIP             (true,  "GossipStage",           "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
     REQUEST_RESPONSE   (false, "RequestResponseStage",  "request",  FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedLowSignalStage),
     ANTI_ENTROPY       (false, "AntiEntropyStage",      "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
@@ -59,7 +59,6 @@ public enum Stage
     PAXOS_REPAIR       (false, "PaxosRepairStage",      "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     INTERNAL_METADATA  (false, "InternalMetadataStage", "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     FETCH_LOG          (false, "MetadataFetchLogStage", "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
-    ACCORD_RANGE_LOADER(false, "AccordRangeLoader",     "internal", () -> 4,                                         null,                                            Stage::multiThreadedStage),
     ;
     public final String jmxName;
     private final Supplier<ExecutorPlus> executorSupplier;

src/java/org/apache/cassandra/config/AccordSpec.java

@@ -21,11 +21,14 @@ package org.apache.cassandra.config;
 import java.util.concurrent.TimeUnit;
 
 import accord.primitives.TxnId;
+import accord.utils.Invariants;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 
 import static accord.primitives.Routable.Domain.Range;
+import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_POOL_PER_SHARD;
+import static org.apache.cassandra.config.AccordSpec.QueueSubmissionModel.SYNC;
 
 public class AccordSpec
 {
@@ -35,18 +38,99 @@ public class AccordSpec
 
     public volatile boolean enable_journal_compaction = true;
 
-    public volatile OptionaldPositiveInt shard_count = OptionaldPositiveInt.UNDEFINED;
+    public enum QueueShardModel
+    {
+        /**
+         * Same number of threads as queue shards, but the shard lock is held only while managing the queue,
+         * so that submitting threads may queue load/save work.
+         *
+         * The global READ and WRITE stages are used for IO.
+         */
+        THREAD_PER_SHARD,
+
+        /**
+         * Same number of threads as shards, and the shard lock is held for the duration of serving requests.
+         * The global READ and WRITE stages are used for IO.
+         */
+        THREAD_PER_SHARD_SYNC_QUEUE,
+
+        /**
+         * More threads than shards. Threads update transaction state as well as performing IO, minimising context switching.
+         * Fewer shards is generally better, until queue-contention is encountered.
+         */
+        THREAD_POOL_PER_SHARD,
+
+        /**
+         * More threads than shards. Threads update transaction state only, relying on READ and WRITE stages for IO.
+         * Fewer shards is generally better, until queue-contention is encountered.
+         */
+        THREAD_POOL_PER_SHARD_EXCLUDES_IO,
+    }
+
+    public enum QueueSubmissionModel
+    {
+        /**
+         * The queue workers and all submissions require ownership of the lock.
+         */
+        SYNC,
+
+        /**
+         * The queue workers and some submissions require ownership of the lock.
+         * That is, if the lock is available on submission we take it; if it is not we try to guarantee that
+         * another thread will witness the work submission promptly, but if we cannot we wait for the lock
+         * to ensure work is scheduled.
+         */
+        SEMI_SYNC,
+
+        /**
+         * The queue workers only require ownership of the lock, submissions happens fully asynchronously.
+         */
+        ASYNC,
+
+        /**
+         * The queue is backed by submission to a single-threaded plain executor.
+         * This implementation does not honur the sharding model option.
+         *
+         * Note: this isn't intended to be used by real clusters.
+         */
+        EXEC_ST
+    }
+
+    public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD;
+    public QueueSubmissionModel queue_submission_model = SYNC;
+
+    /**
+     * The number of queue (and cache) shards.
+     */
+    public volatile OptionaldPositiveInt queue_shard_count = OptionaldPositiveInt.UNDEFINED;
+
+    /**
+     * The target number of command stores to create per topology shard.
+     * This determines the amount of execution parallelism possible for a given table/shard on the host.
+     * More shards means more parallelism, but more state.
+     *
+     * TODO (expected): make this a table property
+     * TODO (expected): adjust this by proportion of ring
+     */
+    public volatile OptionaldPositiveInt command_store_shard_count = OptionaldPositiveInt.UNDEFINED;
+
+    public volatile OptionaldPositiveInt max_queued_loads = OptionaldPositiveInt.UNDEFINED;
+    public volatile OptionaldPositiveInt max_queued_range_loads = OptionaldPositiveInt.UNDEFINED;
+
+    public DataStorageSpec.LongMebibytesBound cache_size = null;
+    public DataStorageSpec.LongMebibytesBound working_set_size = null;
+    public boolean shrink_cache_entries_before_eviction = true;
 
     // TODO (expected): we should be able to support lower recover delays, at least for txns
     public volatile DurationSpec.IntMillisecondsBound recover_delay = new DurationSpec.IntMillisecondsBound(5000);
-    public volatile DurationSpec.IntMillisecondsBound range_sync_recover_delay = new DurationSpec.IntMillisecondsBound("5m");
+    public volatile DurationSpec.IntMillisecondsBound range_syncpoint_recover_delay = new DurationSpec.IntMillisecondsBound("5m");
     public String slowPreAccept = "30ms <= p50*2 <= 100ms";
     public String slowRead = "30ms <= p50*2 <= 100ms";
 
     public long recoveryDelayFor(TxnId txnId, TimeUnit unit)
     {
         if (txnId.isSyncPoint() && txnId.is(Range))
-            return range_sync_recover_delay.to(unit);
+            return range_syncpoint_recover_delay.to(unit);
         return recover_delay.to(unit);
     }
 
@@ -63,7 +147,7 @@ public class AccordSpec
 
     public DurationSpec.IntMillisecondsBound barrier_max_backoff = new DurationSpec.IntMillisecondsBound("10m");
 
-    public DurationSpec.IntMillisecondsBound range_barrier_timeout = new DurationSpec.IntMillisecondsBound("2m");
+    public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("2m");
 
     public volatile DurationSpec.IntSecondsBound fast_path_update_delay = new DurationSpec.IntSecondsBound("60m");
 
@@ -92,7 +176,7 @@ public class AccordSpec
      * default transactional mode for tables created by this node when no transactional mode has been specified in the DDL
      */
     public TransactionalMode default_transactional_mode = TransactionalMode.off;
-    public boolean ephemeralReadEnabled = false;
+    public boolean ephemeralReadEnabled = true;
     public boolean state_cache_listener_jfr_enabled = true;
     public final JournalSpec journal = new JournalSpec();
     public final MinEpochRetrySpec minEpochSyncRetry = new MinEpochRetrySpec();
@@ -110,9 +194,22 @@ public class AccordSpec
         public int segmentSize = 32 << 20;
         public FailurePolicy failurePolicy = FailurePolicy.STOP;
         public FlushMode flushMode = FlushMode.PERIODIC;
-        public DurationSpec.IntMillisecondsBound flushPeriod; // pulls default from 'commitlog_sync_period'
-        public DurationSpec.IntMillisecondsBound periodicFlushLagBlock = new DurationSpec.IntMillisecondsBound("1500ms");
+        public volatile DurationSpec flushPeriod; // pulls default from 'commitlog_sync_period'
+        public DurationSpec periodicFlushLagBlock = new DurationSpec.IntMillisecondsBound("1500ms");
         public DurationSpec.IntMillisecondsBound compactionPeriod = new DurationSpec.IntMillisecondsBound("60000ms");
+        private volatile long flushCombinedBlockPeriod = Long.MIN_VALUE;
+
+        public void setFlushPeriod(DurationSpec newFlushPeriod)
+        {
+            flushPeriod = newFlushPeriod;
+            flushCombinedBlockPeriod = Long.MIN_VALUE;
+        }
+
+        public void setPeriodicFlushLagBlock(DurationSpec newPeriodicFlushLagBlock)
+        {
+            periodicFlushLagBlock = newPeriodicFlushLagBlock;
+            flushCombinedBlockPeriod = Long.MIN_VALUE;
+        }
 
         @Override
         public int segmentSize()
@@ -139,24 +236,32 @@ public class AccordSpec
         }
 
         @Override
-        public int compactionPeriodMillis()
+        public long compactionPeriod(TimeUnit unit)
         {
-            return compactionPeriod.toMilliseconds();
+            return compactionPeriod.to(unit);
         }
 
         @JsonIgnore
         @Override
-        public int flushPeriodMillis()
+        public long flushPeriod(TimeUnit units)
         {
-            return flushPeriod == null ? DatabaseDescriptor.getCommitLogSyncPeriod()
-                                       : flushPeriod.toMilliseconds();
+            return flushPeriod.to(units);
         }
 
         @JsonIgnore
         @Override
-        public int periodicFlushLagBlock()
+        public long periodicBlockPeriod(TimeUnit units)
         {
-            return periodicFlushLagBlock.toMilliseconds();
+            long nanos = flushCombinedBlockPeriod;
+            if (nanos >= 0)
+                return units.convert(nanos, TimeUnit.NANOSECONDS);
+
+            long flushPeriodNanos = flushPeriod(TimeUnit.NANOSECONDS);
+            Invariants.checkState(flushPeriodNanos > 0);
+            nanos = periodicFlushLagBlock.to(TimeUnit.NANOSECONDS) + flushPeriodNanos;
+            // it is possible for this to race and cache the wrong value after an update
+            flushCombinedBlockPeriod = nanos;
+            return nanos;
         }
 
         /**

src/java/org/apache/cassandra/config/Config.java

@@ -501,7 +501,6 @@ public class Config
     public volatile int counter_cache_keys_to_save = Integer.MAX_VALUE;
 
     public DataStorageSpec.LongMebibytesBound paxos_cache_size = null;
-    public DataStorageSpec.LongMebibytesBound accord_cache_size = null;
 
     public DataStorageSpec.LongMebibytesBound consensus_migration_cache_size = null;
 

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -164,6 +164,7 @@ import static org.apache.cassandra.db.ConsistencyLevel.ONE;
 import static org.apache.cassandra.db.ConsistencyLevel.QUORUM;
 import static org.apache.cassandra.io.util.FileUtils.ONE_GIB;
 import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
+import static org.apache.cassandra.journal.Params.FlushMode.PERIODIC;
 import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome;
 
 public class DatabaseDescriptor
@@ -226,6 +227,7 @@ public class DatabaseDescriptor
     private static long keyCacheSizeInMiB;
     private static long paxosCacheSizeInMiB;
     private static long accordCacheSizeInMiB;
+    private static long accordWorkingSetSizeInMiB;
     private static long consensusMigrationCacheSizeInMiB;
     private static long counterCacheSizeInMiB;
     private static long indexSummaryCapacityInMiB;
@@ -611,6 +613,16 @@ public class DatabaseDescriptor
             logger.debug("Syncing log with a period of {}", conf.commitlog_sync_period.toString());
         }
 
+        if (conf.accord.journal.flushPeriod == null)
+        {
+            conf.accord.journal.flushPeriod = conf.commitlog_sync_period;
+            if (conf.accord.journal.flushMode == PERIODIC && conf.commitlog_sync_period.toMilliseconds() == 0)
+            {
+                logger.warn("Accord journal is configured in periodic mode, while Cassandra commit log is configured in {} mode", conf.commitlog_sync);
+                conf.accord.journal.flushPeriod = conf.accord.journal.periodicFlushLagBlock;
+            }
+        }
+
         /* evaluate the DiskAccessMode Config directive, which also affects indexAccessMode selection */
         if (conf.disk_access_mode == DiskAccessMode.auto || conf.disk_access_mode == DiskAccessMode.mmap_index_only)
         {
@@ -964,18 +976,32 @@ public class DatabaseDescriptor
 
         try
         {
-            // if paxosCacheSizeInMiB option was set to "auto" then size of the cache should be "max(10% of Heap (in MB), 1MB)
-            accordCacheSizeInMiB = (conf.accord_cache_size == null)
+            // if accordCacheSizeInMiB option was set to "auto" then size of the cache should be "max(10% of Heap (in MB), 1MB)
+            accordCacheSizeInMiB = (conf.accord.cache_size == null)
                                   ? Math.max(1, (int) ((Runtime.getRuntime().totalMemory() * 0.10) / 1024 / 1024))
-                                  : conf.accord_cache_size.toMebibytes();
+                                  : conf.accord.cache_size.toMebibytes();
 
             if (accordCacheSizeInMiB < 0)
                 throw new NumberFormatException(); // to escape duplicating error message
         }
         catch (NumberFormatException e)
         {
-            throw new ConfigurationException("paxos_cache_size option was set incorrectly to '"
-                                             + conf.paxos_cache_size + "', supported values are <integer> >= 0.", false);
+            throw new ConfigurationException("accord.cache_size option was set incorrectly to '"
+                                             + conf.accord.cache_size + "', supported values are <integer> >= 0.", false);
+        }
+
+        try
+        {
+            // if accordWorkingSetSizeInMiB option was set to "auto" then size of the working set should be "max(5% of Heap (in MB), 1MB)
+            // if negative, there is no limit
+            accordWorkingSetSizeInMiB = (conf.accord.working_set_size == null)
+                                  ? Math.max(1, (int) ((Runtime.getRuntime().totalMemory() * 0.05) / 1024 / 1024))
+                                  : conf.accord.working_set_size.toMebibytes();
+        }
+        catch (NumberFormatException e)
+        {
+            throw new ConfigurationException("accord.working_set_size option was set incorrectly to '"
+                                             + conf.accord.working_set_size + "', supported values are <integer> >= 0.", false);
         }
 
         try
@@ -2729,7 +2755,7 @@ public class DatabaseDescriptor
         conf.concurrent_materialized_view_writes = concurrent_materialized_view_writes;
     }
 
-    public static int getConcurrentAccordOps()
+    public static int getAccordConcurrentOps()
     {
         return conf.concurrent_accord_operations;
     }
@@ -3659,41 +3685,6 @@ public class DatabaseDescriptor
         return conf.paxos_topology_repair_strict_each_quorum;
     }
 
-    public static AccordSpec getAccord()
-    {
-        return conf == null ? null : conf.accord;
-    }
-
-    public static AccordSpec.TransactionalRangeMigration getTransactionalRangeMigration()
-    {
-        return conf.accord.range_migration;
-    }
-
-    public static void setTransactionalRangeMigration(AccordSpec.TransactionalRangeMigration val)
-    {
-        conf.accord.range_migration = Preconditions.checkNotNull(val);
-    }
-
-    public static int getAccordBarrierRetryAttempts()
-    {
-        return conf.accord.barrier_retry_attempts;
-    }
-
-    public static long getAccordBarrierRetryInitialBackoffMillis()
-    {
-        return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds();
-    }
-
-    public static long getAccordBarrierRetryMaxBackoffMillis()
-    {
-        return conf.accord.barrier_max_backoff.toMilliseconds();
-    }
-
-    public static long getAccordRangeBarrierTimeoutNanos()
-    {
-        return conf.accord.range_barrier_timeout.to(TimeUnit.NANOSECONDS);
-    }
-
     public static TransactionalMode defaultTransactionalMode()
     {
         return conf.accord.default_transactional_mode;
@@ -4296,6 +4287,11 @@ public class DatabaseDescriptor
         return accordCacheSizeInMiB;
     }
 
+    public static long getAccordWorkingSetSizeInMiB()
+    {
+        return accordWorkingSetSizeInMiB;
+    }
+
     public static long getConsensusMigrationCacheSizeInMiB()
     {
         return consensusMigrationCacheSizeInMiB;
@@ -5313,6 +5309,42 @@ public class DatabaseDescriptor
         }
     }
 
+
+    public static AccordSpec getAccord()
+    {
+        return conf.accord;
+    }
+
+    public static AccordSpec.TransactionalRangeMigration getTransactionalRangeMigration()
+    {
+        return conf.accord.range_migration;
+    }
+
+    public static void setTransactionalRangeMigration(AccordSpec.TransactionalRangeMigration val)
+    {
+        conf.accord.range_migration = Preconditions.checkNotNull(val);
+    }
+
+    public static int getAccordBarrierRetryAttempts()
+    {
+        return conf.accord.barrier_retry_attempts;
+    }
+
+    public static long getAccordBarrierRetryInitialBackoffMillis()
+    {
+        return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds();
+    }
+
+    public static long getAccordBarrierRetryMaxBackoffMillis()
+    {
+        return conf.accord.barrier_max_backoff.toMilliseconds();
+    }
+
+    public static long getAccordRangeSyncPointTimeoutNanos()
+    {
+        return conf.accord.range_syncpoint_timeout.to(TimeUnit.NANOSECONDS);
+    }
+
     public static boolean getAccordTransactionsEnabled()
     {
         return conf.accord.enabled;
@@ -5323,9 +5355,69 @@ public class DatabaseDescriptor
         conf.accord.enabled = b;
     }
 
-    public static int getAccordShardCount()
+    public static AccordSpec.QueueShardModel getAccordQueueShardModel()
     {
-        return conf.accord.shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+        return conf.accord.queue_shard_model;
+    }
+
+    public static AccordSpec.QueueSubmissionModel getAccordQueueSubmissionModel()
+    {
+        return conf.accord.queue_submission_model;
+    }
+
+    public static int getAccordQueueShardCount()
+    {
+        switch (getAccordQueueShardModel())
+        {
+            default: throw new AssertionError("Unhandled queue_shard_model: " + conf.accord.queue_shard_model);
+            case THREAD_PER_SHARD:
+            case THREAD_PER_SHARD_SYNC_QUEUE:
+                return conf.accord.queue_shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+            case THREAD_POOL_PER_SHARD:
+            case THREAD_POOL_PER_SHARD_EXCLUDES_IO:
+                int defaultMax = getAccordQueueSubmissionModel() == AccordSpec.QueueSubmissionModel.SYNC ? 8 : 4;
+                return conf.accord.queue_shard_count.or(Math.min(defaultMax, DatabaseDescriptor.getAvailableProcessors()));
+        }
+    }
+
+    public static int getAccordCommandStoreShardCount()
+    {
+        return conf.accord.command_store_shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+    }
+
+    public static int getAccordMaxQueuedLoadCount()
+    {
+        return conf.accord.max_queued_loads.or(getAccordConcurrentOps());
+    }
+
+    public static int getAccordMaxQueuedRangeLoadCount()
+    {
+        return conf.accord.max_queued_range_loads.or(Math.max(4, getAccordConcurrentOps() / 4));
+    }
+
+    public static boolean getAccordCacheShrinkingOn()
+    {
+        return conf.accord.shrink_cache_entries_before_eviction;
+    }
+
+    public static long getAccordRecoverDelay(TimeUnit units)
+    {
+        return conf.accord.recover_delay.to(units);
+    }
+
+    public static void setAccordRecoverDelay(long time, TimeUnit units)
+    {
+        conf.accord.recover_delay = new IntMillisecondsBound(time, units);
+    }
+
+    public static long getAccordRangeSyncPointRecoverDelay(TimeUnit units)
+    {
+        return conf.accord.range_syncpoint_recover_delay.to(units);
+    }
+
+    public static void setAccordRangeSyncPointRecoverDelay(long time, TimeUnit units)
+    {
+        conf.accord.range_syncpoint_recover_delay = new IntMillisecondsBound(time, units);
     }
 
     public static long getAccordFastPathUpdateDelayMillis()

src/java/org/apache/cassandra/db/compaction/CompactionIterator.java

@@ -35,6 +35,7 @@ import com.google.common.collect.Ordering;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import accord.api.Agent;
 import accord.local.Cleanup;
 import accord.local.CommandStores;
 import accord.local.CommandStores.RangesForEpoch;
@@ -103,6 +104,7 @@ import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.SavedCommand;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.paxos.PaxosRepairHistory;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
@@ -805,6 +807,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
     class AccordCommandsPurger extends AbstractPurger
     {
+        final Agent agent;
         final Int2ObjectHashMap<RedundantBefore> redundantBefores;
         final Int2ObjectHashMap<DurableBefore> durableBefores;
         final Int2ObjectHashMap<RangesForEpoch> ranges;
@@ -814,7 +817,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
         AccordCommandsPurger(Supplier<IAccordService> accordService)
         {
-            IAccordService.CompactionInfo compactionInfo = accordService.get().getCompactionInfo();
+            IAccordService service = accordService.get();
+            IAccordService.CompactionInfo compactionInfo = service.getCompactionInfo();
+            this.agent = service.agent();
             this.redundantBefores = compactionInfo.redundantBefores;
             this.ranges = compactionInfo.ranges;
             this.durableBefores = compactionInfo.durableBefores;
@@ -852,7 +857,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
             if (saveStatus.is(Status.Invalidated))
                 return saveStatusOnly(saveStatus, row, nowInSec);
 
-            Cleanup cleanup = shouldCleanupPartial(txnId, saveStatus, durability, participants,
+            Cleanup cleanup = shouldCleanupPartial(agent, txnId, saveStatus, durability, participants,
                                                    redundantBefore, durableBefore);
             switch (cleanup)
             {
@@ -1019,6 +1024,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         final ColumnMetadata recordColumn;
         final ColumnMetadata versionColumn;
         final AccordService service;
+        final AccordAgent agent;
 
         JournalKey key = null;
         Object builder = null;
@@ -1036,6 +1042,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
             userVersion = service.journalConfiguration().userVersion();
             IAccordService.CompactionInfo compactionInfo = service.getCompactionInfo();
 
+            this.agent = service.agent();
             this.redundantBefores = compactionInfo.redundantBefores;
             this.ranges = compactionInfo.ranges;
             this.durableBefores = compactionInfo.durableBefores;
@@ -1102,7 +1109,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
                 RedundantBefore redundantBefore = redundantBefores.get(key.commandStoreId);
                 DurableBefore durableBefore = durableBefores.get(key.commandStoreId);
-                Cleanup cleanup = commandBuilder.shouldCleanup(redundantBefore, durableBefore);
+                Cleanup cleanup = commandBuilder.shouldCleanup(agent, redundantBefore, durableBefore);
                 if (cleanup == ERASE)
                     return PartitionUpdate.fullPartitionDelete(metadata(), partition.partitionKey(), maxSeenTimestamp, nowInSec).unfilteredIterator();
 

src/java/org/apache/cassandra/db/marshal/AbstractType.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -30,6 +31,7 @@ import java.util.Objects;
 
 import javax.annotation.Nullable;
 
+import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.ColumnSpecification;
@@ -739,6 +741,39 @@ public abstract class AbstractType<T> implements Comparator<ByteBuffer>, Assignm
             throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement asComparableBytes");
     }
 
+    protected static final FastThreadLocal<byte[]> tmpFlattenBuffer = new FastThreadLocal<>();
+    public static byte[] flattenByteSource(ByteSource source)
+    {
+        byte[] tmpBytes = tmpFlattenBuffer.get();
+        byte[] bytes = tmpBytes;
+        if (bytes == null) bytes = new byte[16];
+        int c = 0;
+        while (true)
+        {
+            int b = source.next();
+            if (b == ByteSource.END_OF_STREAM)
+                break;
+
+            if (c == bytes.length)
+                bytes = Arrays.copyOf(bytes, c * 2);
+
+            bytes[c++] = (byte)b;
+        }
+
+        byte[] result = Arrays.copyOf(bytes, c);
+        if (bytes != tmpBytes) tmpFlattenBuffer.set(bytes);
+        return result;
+    }
+
+    public <V> byte[] asFlatComparableBytes(ValueAccessor<V> accessor, V value, ByteComparable.Version version)
+    {
+        ByteSource source = asComparableBytes(accessor, value, version);
+        if (source == null)
+            return null;
+
+        return flattenByteSource(source);
+    }
+
     public final ByteSource asComparableBytes(ByteBuffer byteBuffer, ByteComparable.Version version)
     {
         return asComparableBytes(ByteBufferAccessor.instance, byteBuffer, version);

src/java/org/apache/cassandra/db/marshal/CompositeType.java

@@ -29,6 +29,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import accord.utils.Invariants;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.serializers.BytesSerializer;
@@ -39,8 +40,15 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
+import static accord.utils.Invariants.Paranoia.CONSTANT;
+import static accord.utils.Invariants.Paranoia.LINEAR;
+import static accord.utils.Invariants.ParanoiaCostFactor.LOW;
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.END_OF_STREAM;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_NULL;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.TERMINATOR;
 
 /*
  * The encoding of a CompositeType column name should be:
@@ -250,7 +258,64 @@ public class CompositeType extends AbstractCompositeType
         if (i * 2 + 1 < srcs.length)
             srcs = Arrays.copyOfRange(srcs, 0, i * 2 + 1);
 
-        return ByteSource.withTerminatorMaybeLegacy(version, ByteSource.END_OF_STREAM, srcs);
+        return ByteSource.withTerminatorMaybeLegacy(version, END_OF_STREAM, srcs);
+    }
+
+    @Override
+    public <V> byte[] asFlatComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+    {
+        if (data == null || accessor.isEmpty(data))
+            return null;
+
+        byte[] tmpBytes = tmpFlattenBuffer.get();
+        byte[] bytes = tmpBytes;
+        if (bytes == null) bytes = new byte[16];
+
+        int c = 0;
+        int length = accessor.size(data);
+
+        // statics go first
+        boolean isStatic = readIsStaticInternal(data, accessor);
+        int offset = startingOffsetInternal(isStatic);
+        bytes[c++] = (byte) (isStatic ? NEXT_COMPONENT_NULL : NEXT_COMPONENT);
+        bytes[c++] = (byte) (NEXT_COMPONENT);
+
+        int i = 0;
+        byte lastEoc = 0;
+        while (offset < length)
+        {
+            // Only the end-of-component byte of the last component of this composite can be non-zero, so the
+            // component before can't have a non-zero end-of-component byte.
+            assert lastEoc == 0 : lastEoc;
+
+            int componentLength = accessor.getUnsignedShort(data, offset);
+            offset += 2;
+            ByteSource tmp = types.get(i).asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version);
+            while (true)
+            {
+                int b = tmp.next();
+                if (b == END_OF_STREAM) break;
+
+                if (c == bytes.length) bytes = Arrays.copyOf(bytes, c * 2);
+                bytes[c++] = (byte)b;
+            }
+            offset += componentLength;
+            lastEoc = accessor.getByte(data, offset);
+            offset += 1;
+            if (c == bytes.length) bytes = Arrays.copyOf(bytes, c * 2);
+            bytes[c++] = (byte) NEXT_COMPONENT;
+            bytes[c++] = (byte) (lastEoc & 0xFF ^ 0x80); // end-of-component also takes part in comparison as signed byte
+            bytes[c++] = (byte) (offset < length ? NEXT_COMPONENT : version == Version.LEGACY ? END_OF_STREAM : TERMINATOR);
+            ++i;
+        }
+
+        byte[] result = Arrays.copyOf(bytes, c);
+        if (bytes != tmpBytes) tmpFlattenBuffer.set(bytes);
+        byte[] test = super.asFlatComparableBytes(accessor, data, version);
+        if (Invariants.isParanoid() && Invariants.testParanoia(LINEAR, CONSTANT, LOW)) Invariants.checkState(Arrays.equals(test, result));
+        V roundtrip = fromComparableBytes(accessor, ByteSource.peekable(ByteSource.of(result, version)), version);
+        Invariants.checkState(accessor.compare(data, roundtrip, accessor) == 0);
+        return result;
     }
 
     @Override

src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java

@@ -40,6 +40,8 @@ import org.github.jamm.Unmetered;
 
 public abstract class AbstractMemtable implements Memtable
 {
+    private static final AtomicLong nextId = new AtomicLong();
+
     private final AtomicReference<LifecycleTransaction> flushTransaction = new AtomicReference<>(null);
     protected final AtomicLong currentOperations = new AtomicLong(0);
     protected final ColumnsCollector columnsCollector;
@@ -48,6 +50,7 @@ public abstract class AbstractMemtable implements Memtable
     protected AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE);
     // The smallest local deletion time for all partitions in this memtable
     protected AtomicLong minLocalDeletionTime = new AtomicLong(Long.MAX_VALUE);
+    private final long id = nextId.incrementAndGet();
     // Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not
     // correct for their usage.
 
@@ -80,6 +83,12 @@ public abstract class AbstractMemtable implements Memtable
         return currentOperations.get();
     }
 
+    @Override
+    public long getMemtableId()
+    {
+        return id;
+    }
+
     @Override
     public long getMinTimestamp()
     {

src/java/org/apache/cassandra/db/memtable/Memtable.java

@@ -179,6 +179,11 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
 
     // Main write and read operations
 
+    default long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        return put(update, indexer, opGroup, false);
+    }
+
     /**
      * Put new data in the memtable. This operation may block until enough memory is available in the memory pool.
      *
@@ -186,12 +191,14 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
      * @param indexer receives information about the update's effect
      * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a
      *                flush to free space.
+     * @param assumeMissing if true, the implementation MAY clone the key and attempt putIfAbsent without first
+     *                      looking for the keys' presence
      *
      * @return the smallest timestamp delta between corresponding rows from existing and update. A
      * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition
      * and those in {@code update}. See CASSANDRA-7979.
      */
-    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup);
+    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing);
 
     // Read operations are provided by the UnfilteredSource interface.
 
@@ -363,6 +370,8 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
      */
     boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition);
 
+    long getMemtableId();
+
     /** Approximate commit log lower bound, <= getCommitLogLowerBound, used as a time stamp for ordering */
     CommitLogPosition getApproximateCommitLogLowerBound();
 

src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java

@@ -138,11 +138,11 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
      *
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
         DecoratedKey key = update.partitionKey();
         MemtableShard shard = shards[boundaries.getShardForKey(key)];
-        return shard.put(key, update, indexer, opGroup);
+        return shard.put(key, update, indexer, opGroup, assumeMissing);
     }
 
     /**
@@ -366,10 +366,10 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
             this.metadata = metadata;
         }
 
-        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
         {
             Cloner cloner = allocator.cloner(opGroup);
-            AtomicBTreePartition previous = partitions.get(key);
+            AtomicBTreePartition previous = assumeMissing ? null : partitions.get(key);
 
             long initialSize = 0;
             if (previous == null)
@@ -504,13 +504,13 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
          *
          * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
          */
-        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
         {
             DecoratedKey key = update.partitionKey();
             MemtableShard shard = shards[boundaries.getShardForKey(key)];
             synchronized (shard)
             {
-                return shard.put(key, update, indexer, opGroup);
+                return shard.put(key, update, indexer, opGroup, assumeMissing);
             }
         }
 

src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java

@@ -114,15 +114,14 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
     @Override
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
-        Cloner cloner = allocator.cloner(opGroup);
-        AtomicBTreePartition previous = partitions.get(update.partitionKey());
-
         long initialSize = 0;
+        Cloner cloner = allocator.cloner(opGroup);
+        AtomicBTreePartition previous = assumeMissing ? null : partitions.get(update.partitionKey());
         if (previous == null)
         {
-            final DecoratedKey cloneKey = cloner.clone(update.partitionKey());
+            DecoratedKey cloneKey = cloner.clone(update.partitionKey());
             AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator);
             // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent
             previous = partitions.putIfAbsent(cloneKey, empty);

src/java/org/apache/cassandra/db/memtable/TrieMemtable.java

@@ -180,7 +180,7 @@ public class TrieMemtable extends AbstractShardedMemtable
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
     @Override
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
         try
         {

src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java

@@ -36,12 +36,9 @@ import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import accord.local.CommandStores;
 import accord.primitives.Status;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
@@ -57,10 +54,11 @@ import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.AccordCommandStore;
+import org.apache.cassandra.service.accord.AccordCache;
+import org.apache.cassandra.service.accord.AccordCommandStores;
+import org.apache.cassandra.service.accord.AccordExecutor;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordStateCache;
 import org.apache.cassandra.service.accord.CommandStoreTxnBlockedGraph;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
@@ -81,20 +79,20 @@ public class AccordVirtualTables
             return Collections.emptyList();
 
         return List.of(
-            new CommandStoreCache(keyspace),
+            new ExecutorCache(keyspace),
             new MigrationState(keyspace),
             new CoordinationStatus(keyspace),
             new TxnBlockedByTable(keyspace)
         );
     }
 
-    public static final class CommandStoreCache extends AbstractVirtualTable
+    public static final class ExecutorCache extends AbstractVirtualTable
     {
-        private CommandStoreCache(String keyspace)
+        private ExecutorCache(String keyspace)
         {
             super(parse(keyspace,
-                        "Accord Command Store Cache Metrics",
-                        "CREATE TABLE accord_command_store_cache(\n" +
+                        "Accord Executor Cache Metrics",
+                        "CREATE TABLE accord_executor_cache(\n" +
                         "  id int,\n" +
                         "  scope text,\n" +
                         "  queries bigint,\n" +
@@ -107,32 +105,22 @@ public class AccordVirtualTables
         @Override
         public DataSet data()
         {
-            CommandStores stores = ((AccordService) AccordService.instance()).node().commandStores();
-
-            AsyncChain<List<Map<String, AccordStateCache.ImmutableStats>>> statsByStoreChain = stores.map(store -> {
-                Map<String, AccordStateCache.ImmutableStats> snapshots = new HashMap<>(3);
-                AccordCommandStore accordStore = (AccordCommandStore) store.commandStore();
-                snapshots.put(AccordKeyspace.COMMANDS, accordStore.commandCache().statsSnapshot());
-                snapshots.put(AccordKeyspace.COMMANDS_FOR_KEY, accordStore.commandsForKeyCache().statsSnapshot());
-                snapshots.put(AccordKeyspace.TIMESTAMPS_FOR_KEY, accordStore.timestampsForKeyCache().statsSnapshot());
-                return snapshots;
-            });
-
-            List<Map<String, AccordStateCache.ImmutableStats>> statsByStore = AsyncChains.getBlockingAndRethrow(statsByStoreChain);
+            AccordCommandStores stores = (AccordCommandStores) ((AccordService) AccordService.instance()).node().commandStores();
             SimpleDataSet result = new SimpleDataSet(metadata());
-
-            for (int storeID : stores.ids())
+            for (AccordExecutor executor : stores.executors())
             {
-                Map<String, AccordStateCache.ImmutableStats> storeStats = statsByStore.get(storeID);
-                addRow(storeStats.get(AccordKeyspace.COMMANDS), result, storeID, AccordKeyspace.COMMANDS);
-                addRow(storeStats.get(AccordKeyspace.COMMANDS_FOR_KEY), result, storeID, AccordKeyspace.COMMANDS_FOR_KEY);
-                addRow(storeStats.get(AccordKeyspace.TIMESTAMPS_FOR_KEY), result, storeID, AccordKeyspace.TIMESTAMPS_FOR_KEY);
+                Map<String, AccordCache.ImmutableStats> snapshots = new HashMap<>(3);
+                try (AccordExecutor.ExclusiveGlobalCaches cache = executor.lockCaches())
+                {
+                    addRow(cache.commands.statsSnapshot(), result, executor.executorId(), AccordKeyspace.COMMANDS);
+                    addRow(cache.commandsForKey.statsSnapshot(), result, executor.executorId(), AccordKeyspace.COMMANDS_FOR_KEY);
+                    addRow(cache.timestampsForKey.statsSnapshot(), result, executor.executorId(), AccordKeyspace.TIMESTAMPS_FOR_KEY);
+                }
             }
-
             return result;
         }
 
-        private static void addRow(AccordStateCache.ImmutableStats stats, SimpleDataSet result, int storeID, String scope)
+        private static void addRow(AccordCache.ImmutableStats stats, SimpleDataSet result, int storeID, String scope)
         {
             result.row(storeID, scope);
             result.column("queries", stats.queries);

src/java/org/apache/cassandra/dht/LocalPartitioner.java

@@ -186,7 +186,6 @@ public class LocalPartitioner implements IPartitioner
         {
             // todo (tcm); seems partitioner got mutated on alter type (for example) before tcm, now we create a new one - not sure its enough just making sure that its the same type of partitioner
             assert o.getPartitioner().getClass().equals(getPartitioner().getClass()) : String.format("partitioners do not match; %s != %s", getPartitioner(), o.getPartitioner());
-//            assert getPartitioner() == o.getPartitioner() : String.format("partitioners do not match; %s != %s", getPartitioner(), o.getPartitioner());
             return comparator.compare(token, ((LocalToken) o).token);
         }
 

src/java/org/apache/cassandra/hints/HintsBuffer.java

@@ -179,7 +179,19 @@ final class HintsBuffer
         return new Allocation(offset, totalSize, opGroup);
     }
 
-    // allocate bytes in the slab, or return negative if not enough space
+    /**
+     * Allocate bytes in the segment, or return -1 if not enough space. Method ensures that marker bytes
+     * for each allocation (i.e. offset of its end) is written as a 32 bit integer at its beginning, and
+     * that these marker bytes are always written sequentially. In other words, if allocation A has a lower
+     * starting offset than allocation B, A's marker will always be written before the offset for B is returned.
+     *
+     * `allocateOffset` consists of two integers:
+     *    64                 32                0
+     *    | (i32) inProgress | (i32) writtenTo |
+     *
+     *  If inProgress bytes are not zeroes, they contain an unwritten offset. Before allocating any bytes,
+     *  inProgresss bytes need to be written at the writtenTo location in the target buffer.
+     */
     private int allocateBytes(int totalSize)
     {
         long prev = position.getAndAdd(totalSize);

src/java/org/apache/cassandra/hints/HintsService.java

@@ -239,6 +239,14 @@ public final class HintsService implements HintsServiceMBean
         writeExecutor.fsyncWritersBlockingly(stores);
     }
 
+    @VisibleForTesting
+    public void flushAndFsyncBlockingly()
+    {
+        List<HintsStore> stores = catalog.stores().collect(Collectors.toList());
+        writeExecutor.flushBufferPool(bufferPool, stores);
+        writeExecutor.fsyncWritersBlockingly(stores);
+    }
+
     public synchronized void startDispatch()
     {
         if (isShutDown)

src/java/org/apache/cassandra/index/accord/RoutesSearcher.java

@@ -19,12 +19,10 @@
 package org.apache.cassandra.index.accord;
 
 import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Set;
+import java.util.function.Consumer;
 
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
-import org.agrona.collections.ObjectHashSet;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
@@ -48,7 +46,7 @@ import org.apache.cassandra.utils.FBUtilities;
 public class RoutesSearcher
 {
     private final ColumnFamilyStore cfs = Keyspace.open("system_accord").getColumnFamilyStore("commands");
-    private final Index index = cfs.indexManager.getIndexByName("route");;
+    private final Index index = cfs.indexManager.getIndexByName("route");
     private final ColumnMetadata participants = AccordKeyspace.CommandsColumns.participants;
     private final ColumnMetadata store_id = AccordKeyspace.CommandsColumns.store_id;
     private final ColumnMetadata txn_id = AccordKeyspace.CommandsColumns.txn_id;
@@ -101,14 +99,13 @@ public class RoutesSearcher
         }
     }
 
-    public Set<TxnId> intersects(int store, TokenRange range, TxnId minTxnId, Timestamp maxTxnId)
+    public void intersects(int store, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, Consumer<TxnId> forEach)
     {
-        return intersects(store, range.start(), range.end(), minTxnId, maxTxnId);
+        intersects(store, range.start(), range.end(), minTxnId, maxTxnId, forEach);
     }
 
-    public Set<TxnId> intersects(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId)
+    void intersects(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId, Consumer<TxnId> forEach)
     {
-        ObjectHashSet<TxnId> set = new ObjectHashSet<TxnId>();
         try (CloseableIterator<Entry> it = searchKeysAccord(store, start, end))
         {
             while (it.hasNext())
@@ -116,10 +113,9 @@ public class RoutesSearcher
                 Entry next = it.next();
                 if (next.store_id != store) continue; // the index should filter out, but just in case...
                 if (next.txnId.compareTo(minTxnId) >= 0 && next.txnId.compareTo(maxTxnId) < 0)
-                    set.add(next.txnId);
+                    forEach.accept(next.txnId);
             }
         }
-        return set.isEmpty() ? Collections.emptySet() : set;
     }
 
     private static final class Entry

src/java/org/apache/cassandra/io/util/DataOutputBuffer.java

@@ -60,7 +60,7 @@ public class DataOutputBuffer extends BufferedDataOutputStreamPlus
      * Scratch buffers used mostly for serializing in memory. It's important to call #close() when finished
      * to keep the memory overhead from being too large in the system.
      */
-    public static final FastThreadLocal<DataOutputBuffer> scratchBuffer = new FastThreadLocal<DataOutputBuffer>()
+    public static final FastThreadLocal<DataOutputBuffer> scratchBuffer = new FastThreadLocal<>()
     {
         @Override
         protected DataOutputBuffer initialValue()

src/java/org/apache/cassandra/journal/ActiveSegment.java

@@ -24,6 +24,7 @@ import java.nio.channels.FileChannel;
 import java.nio.file.StandardOpenOption;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.locks.LockSupport;
 
 import com.codahale.metrics.Timer;
@@ -44,16 +45,18 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     private final OpOrder appendOrder = new OpOrder();
 
     // position in the buffer we are allocating from
-    private volatile int allocateOffset = 0;
-    private static final AtomicIntegerFieldUpdater<ActiveSegment> allocateOffsetUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "allocateOffset");
+    private volatile long allocateOffset = 0;
+    private static final AtomicLongFieldUpdater<ActiveSegment> allocateOffsetUpdater = AtomicLongFieldUpdater.newUpdater(ActiveSegment.class, "allocateOffset");
 
     /*
      * Everything before this offset has been written and flushed.
      */
-    private volatile int lastFlushedOffset = 0;
-    private volatile int lastFsyncOffset = 0;
+    private volatile int writtenTo = 0;
+    private volatile int fsyncedTo = 0;
     @SuppressWarnings("rawtypes")
-    private static final AtomicIntegerFieldUpdater<ActiveSegment> lastFsyncOffsetUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "lastFsyncOffset");
+    private static final AtomicIntegerFieldUpdater<ActiveSegment> writtenToUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "writtenTo");
+    @SuppressWarnings("rawtypes")
+    private static final AtomicIntegerFieldUpdater<ActiveSegment> fsyncedToUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "fsyncedTo");
 
     /*
      * End position of the buffer; initially set to its capacity and
@@ -70,16 +73,16 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     final InMemoryIndex<K> index;
 
     private ActiveSegment(
-        Descriptor descriptor, Params params, SyncedOffsets syncedOffsets, InMemoryIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
+        Descriptor descriptor, Params params, InMemoryIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
     {
-        super(descriptor, syncedOffsets, metadata, keySupport);
+        super(descriptor, metadata, keySupport);
         this.index = index;
         try
         {
             channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE);
             buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, params.segmentSize());
             endOfBuffer = buffer.capacity();
-            selfRef = new Ref<>(this, new Tidier(descriptor, channel, buffer, syncedOffsets));
+            selfRef = new Ref<>(this, new Tidier(descriptor, channel, buffer));
         }
         catch (IOException e)
         {
@@ -87,13 +90,11 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    @SuppressWarnings("resource")
     static <K, V> ActiveSegment<K, V> create(Descriptor descriptor, Params params, KeySupport<K> keySupport)
     {
-        SyncedOffsets syncedOffsets = SyncedOffsets.active(descriptor);
         InMemoryIndex<K> index = InMemoryIndex.create(keySupport);
         Metadata metadata = Metadata.create();
-        return new ActiveSegment<>(descriptor, params, syncedOffsets, index, metadata, keySupport);
+        return new ActiveSegment<>(descriptor, params, index, metadata, keySupport);
     }
 
     @Override
@@ -147,40 +148,40 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     /**
      * Stop writing to this file, flush and close it. Does nothing if the file is already closed.
      */
-    @Override
-    public synchronized void close()
+    public synchronized void close(Journal<K, V> journal)
     {
-        close(true);
+        close(journal, true);
     }
 
     /**
      * @return true if the closed segment was definitely empty, false otherwise
      */
-    private synchronized boolean close(boolean persistComponents)
+    private synchronized boolean close(Journal<K, V> journal, boolean persistComponents)
     {
         boolean isEmpty = discardUnusedTail();
         if (!isEmpty)
         {
-            flush(true);
+            updateWrittenTo();
+            fsync();
             if (persistComponents) persistComponents();
         }
-        release();
+        release(journal);
         return isEmpty;
     }
 
     /**
      * Close and discard a pre-allocated, available segment, that's never been exposed
      */
-    void closeAndDiscard()
+    void closeAndDiscard(Journal<K, V> journal)
     {
-        boolean isEmpty = close(false);
+        boolean isEmpty = close(journal, false);
         if (!isEmpty) throw new IllegalStateException();
         discard();
     }
 
-    void closeAndIfEmptyDiscard()
+    void closeAndIfEmptyDiscard(Journal<K, V> journal)
     {
-        boolean isEmpty = close(true);
+        boolean isEmpty = close(journal, true);
         if (isEmpty) discard();
     }
 
@@ -188,7 +189,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     {
         index.persist(descriptor);
         metadata.persist(descriptor);
-        syncedOffsets.fsync();
         SyncUtil.trySyncDir(descriptor.directory);
     }
 
@@ -199,13 +199,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         descriptor.fileFor(Component.DATA).deleteIfExists();
         descriptor.fileFor(Component.INDEX).deleteIfExists();
         descriptor.fileFor(Component.METADATA).deleteIfExists();
-        descriptor.fileFor(Component.SYNCED_OFFSETS).deleteIfExists();
-    }
-
-    @Override
-    void release()
-    {
-        selfRef.release();
     }
 
     @Override
@@ -220,23 +213,27 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         return selfRef.ref();
     }
 
-    private static final class Tidier implements Tidy
+    @Override
+    public Ref<Segment<K, V>> selfRef()
+    {
+        return selfRef;
+    }
+
+    private static final class Tidier extends Segment.Tidier implements Tidy
     {
         private final Descriptor descriptor;
         private final FileChannel channel;
         private final ByteBuffer buffer;
-        private final SyncedOffsets syncedOffsets;
 
-        Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer, SyncedOffsets syncedOffsets)
+        Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer)
         {
             this.descriptor = descriptor;
             this.channel = channel;
             this.buffer = buffer;
-            this.syncedOffsets = syncedOffsets;
         }
 
         @Override
-        public void tidy()
+        void onUnreferenced()
         {
             FileUtils.clean(buffer);
             try
@@ -247,7 +244,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             {
                 throw new JournalWriteError(descriptor, Component.DATA, e);
             }
-            syncedOffsets.close();
         }
 
         @Override
@@ -257,68 +253,49 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    /*
-     * Flush logic; closing and component flushing
-     */
-
-    boolean shouldFlush()
-    {
-        int allocateOffset = this.allocateOffset;
-        return lastFlushedOffset < allocateOffset;
-    }
-
     public boolean isFlushed(long position)
     {
-        return lastFlushedOffset >= position;
+        return writtenTo >= position;
     }
 
-    public long lastFlushedOffset()
+    public int writtenToAtLeast()
     {
-        return lastFlushedOffset;
+        return writtenTo;
     }
 
-    /**
-     * Possibly force a disk flush for this segment file.
-     * TODO FIXME: calls from outside Flusher + callbacks
-     * @return last synced offset
-     */
-    synchronized int flush(boolean fsync)
+    public int fsyncedTo()
     {
-        int allocateOffset = this.allocateOffset;
-        if (lastFlushedOffset >= allocateOffset)
-            return lastFlushedOffset;
+        return fsyncedTo;
+    }
+
+    public int updateWrittenTo()
+    {
+        int allocatedTo = (int)allocateOffset;
+        if (writtenTo >= allocatedTo)
+            return writtenTo;
 
         waitForModifications();
-        if (fsync)
-        {
-            fsyncInternal();
-            lastFsyncOffsetUpdater.accumulateAndGet(this, allocateOffset, Math::max);
-        }
-        lastFlushedOffset = allocateOffset;
-        int syncedOffset = Math.min(allocateOffset, endOfBuffer);
-        syncedOffsets.mark(syncedOffset, fsync);
-        flushComplete.signalAll();
-        return syncedOffset;
+        return writtenToUpdater.accumulateAndGet(this, allocatedTo, Math::max);
     }
 
     // provides no ordering guarantees
     void fsync()
     {
-        int lastFlushed = lastFlushedOffset;
-        if (lastFsyncOffset >= lastFlushed)
+        int writtenTo = this.writtenTo;
+        if (fsyncedTo >= writtenTo)
             return;
 
         fsyncInternal();
-        syncedOffsets.fsync();
-        lastFsyncOffsetUpdater.accumulateAndGet(this, lastFlushed, Math::max);
+        fsyncedToUpdater.accumulateAndGet(this, writtenTo, Math::max);
+        flushComplete.signalAll();
     }
 
     private void waitForFlush(int position)
     {
-        while (lastFlushedOffset < position)
+        while (fsyncedTo < position)
         {
             WaitQueue.Signal signal = flushComplete.register();
-            if (lastFlushedOffset < position)
+            if (fsyncedTo < position)
                 signal.awaitThrowUncheckedOnInterrupt();
             else
                 signal.cancel();
@@ -346,12 +323,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    boolean isFullyFlushed()
-    {
-        int allocateOffset = this.allocateOffset;
-        return lastFsyncOffset >= allocateOffset;
-    }
-
     /**
      * Ensures no more of this segment is writeable, by allocating any unused section at the end
      * and marking it discarded void discartUnusedTail()
@@ -364,10 +335,10 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         {
             while (true)
             {
-                int prev = allocateOffset;
+                long prev = completeInProgress();
                 int next = endOfBuffer + 1;
 
-                if (prev >= next)
+                if ((int)prev >= next)
                 {
                     // already stopped allocating, might also be closed
                     assert buffer == null || prev == buffer.capacity() + 1;
@@ -377,10 +348,11 @@ final class ActiveSegment<K, V> extends Segment<K, V>
                 if (allocateOffsetUpdater.compareAndSet(this, prev, next))
                 {
                     // stopped allocating now; can only succeed once, no further allocation or discardUnusedTail can succeed
-                    endOfBuffer = prev;
+                    endOfBuffer = (int)prev;
                     assert buffer != null && next == buffer.capacity() + 1;
                     return prev == 0;
                 }
+                LockSupport.parkNanos(1);
             }
         }
     }
@@ -414,7 +386,8 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     private int totalEntrySize(Set<Integer> hosts, int recordSize)
     {
         return EntrySerializer.fixedEntrySize(keySupport, descriptor.userVersion)
-             + EntrySerializer.variableEntrySize(hosts.size(), recordSize);
+             + EntrySerializer.variableEntrySize(hosts.size())
+               + recordSize;
     }
 
     // allocate bytes in the segment, or return -1 if not enough space
@@ -422,16 +395,30 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     {
         while (true)
         {
-            int prev = allocateOffset;
-            int next = prev + size;
+            long prev = maybeCompleteInProgress();
+            if (prev < 0)
+            {
+                LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+                continue;
+            }
+
+            long next = prev + size;
             if (next >= endOfBuffer)
                 return -1;
-            if (allocateOffsetUpdater.compareAndSet(this, prev, next))
+
+            // TODO (expected): if we write a "safe shutdown" marker we don't need this,
+            //  but this provides safe restart in the event the process terminates abruptly but the host remains stable
+            long inProgress = prev | (next << 32);
+            if (!allocateOffsetUpdater.compareAndSet(this, prev, inProgress))
             {
-                assert buffer != null;
-                return prev;
+                LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+                continue;
             }
-            LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+
+            assert buffer != null;
+            buffer.putInt((int)prev, (int)next);
+            allocateOffsetUpdater.compareAndSet(this, inProgress, next);
+            return (int) prev;
         }
     }
 
@@ -450,14 +437,13 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             this.length = length;
         }
 
-        RecordPointer write(K id, ByteBuffer record, Set<Integer> hosts)
+        void write(K id, ByteBuffer record, Set<Integer> hosts)
         {
-            try (BufferedDataOutputStreamPlus out = new DataOutputBufferFixed(buffer))
+            try
             {
-                EntrySerializer.write(id, record, hosts, keySupport, out, descriptor.userVersion);
-                index.update(id, start, length);
+                EntrySerializer.write(id, record, hosts, keySupport, buffer, descriptor.userVersion);
                 metadata.update(hosts);
-                return new RecordPointer(descriptor.timestamp, start);
+                index.update(id, start, length);
             }
             catch (IOException e)
             {
@@ -472,9 +458,9 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         // Variant of write that does not allocate/return a record pointer
         void writeInternal(K id, ByteBuffer record, Set<Integer> hosts)
         {
-            try (BufferedDataOutputStreamPlus out = new DataOutputBufferFixed(buffer))
+            try
             {
-                EntrySerializer.write(id, record, hosts, keySupport, out, descriptor.userVersion);
+                EntrySerializer.write(id, record, hosts, keySupport, buffer, descriptor.userVersion);
                 index.update(id, start, length);
                 metadata.update(hosts);
             }
@@ -488,12 +474,48 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             }
         }
 
-        void awaitFlush(Timer waitingOnFlush)
+        void awaitDurable(Timer waitingOnFlush)
         {
             try (Timer.Context ignored = waitingOnFlush.time())
             {
                 waitForFlush(start);
             }
         }
+
+        boolean isFsynced()
+        {
+            return fsyncedTo >= start + length;
+        }
+
+        Descriptor descriptor()
+        {
+            return descriptor;
+        }
+
+        int start()
+        {
+            return start;
+        }
+    }
+
+    private int maybeCompleteInProgress()
+    {
+        long cur = allocateOffset;
+        int inProgress = (int) (cur >>> 32);
+        if (inProgress == 0) return (int) cur;
+        // finish up the in-progress allocation
+        buffer.putInt((int)cur, inProgress);
+        if (!allocateOffsetUpdater.compareAndSet(this, cur, inProgress))
+            return -1;
+
+        return inProgress;
+    }
+
+    private int completeInProgress()
+    {
+        int result = maybeCompleteInProgress();
+        while (result < 0)
+            result = maybeCompleteInProgress();
+        return result;
     }
 }

src/java/org/apache/cassandra/journal/Compactor.java

@@ -46,7 +46,7 @@ public final class Compactor<K, V> implements Runnable, Shutdownable
     synchronized void start()
     {
         if (journal.params.enableCompaction())
-            schedule(journal.params.compactionPeriodMillis(), TimeUnit.MILLISECONDS);
+            schedule(journal.params.compactionPeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
     }
 
     private synchronized void schedule(long period, TimeUnit units)
@@ -85,7 +85,7 @@ public final class Compactor<K, V> implements Runnable, Shutdownable
 
             journal.replaceCompactedSegments(toCompact, newSegments);
             for (StaticSegment<K, V> segment : toCompact)
-                segment.discard();
+                segment.discard(journal);
         }
         catch (IOException e)
         {

src/java/org/apache/cassandra/journal/Component.java

@@ -17,15 +17,19 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.List;
+
+import static accord.utils.SortedArrays.SortedArrayList.ofSorted;
+
 enum Component
 {
     DATA           ("data"),
     INDEX          ("indx"),
-    METADATA       ("meta"),
-    SYNCED_OFFSETS ("sync");
+    METADATA       ("meta");
     //OFFSET_MAP     (".offs"),
     //INVLALIDATIONS (".invl");
 
+    public static final List<Component> VALUES = ofSorted(values());
     final String extension;
 
     Component(String extension)

src/java/org/apache/cassandra/journal/EntrySerializer.java

@@ -23,17 +23,13 @@ import java.nio.ByteBuffer;
 import java.util.Set;
 import java.util.zip.CRC32;
 
+import accord.utils.Invariants;
 import org.agrona.collections.IntHashSet;
 import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Crc;
 
 import static org.apache.cassandra.journal.Journal.validateCRC;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumShort;
 
 public final class EntrySerializer
 {
@@ -41,175 +37,172 @@ public final class EntrySerializer
                           ByteBuffer record,
                           Set<Integer> hosts,
                           KeySupport<K> keySupport,
-                          DataOutputPlus out,
+                          ByteBuffer out,
                           int userVersion)
     throws IOException
     {
-        CRC32 crc = Crc.crc32();
+        int start = out.position();
+        int totalSize = out.getInt() - start;
+        Invariants.checkState(totalSize == out.remaining() + TypeSizes.INT_SIZE);
+        Invariants.checkState(totalSize == record.remaining() + fixedEntrySize(keySupport, userVersion) + variableEntrySize(hosts.size()));
 
         keySupport.serialize(key, out, userVersion);
-        keySupport.updateChecksum(crc, key, userVersion);
+        out.putShort((short)hosts.size());
 
-        out.writeShort(hosts.size());
-        updateChecksumShort(crc, (short) hosts.size());
-
-        int recordSize = record.remaining();
-        out.writeInt(recordSize);
-        updateChecksumInt(crc, recordSize);
-
-        out.writeInt((int) crc.getValue());
+        int fixedCrcPosition = out.position();
+        out.position(fixedCrcPosition + TypeSizes.INT_SIZE);
 
         for (int host : hosts)
-        {
-            out.writeInt(host);
-            updateChecksumInt(crc, host);
-        }
+            out.putInt(host);
 
-        out.write(record);
-        Crc.updateCrc32(crc, record, record.position(), record.limit());
+        int recordSize = record.remaining();
+        int recordEnd = out.position() + recordSize;
+        Invariants.checkState(out.limit() == recordEnd + TypeSizes.INT_SIZE);
+        ByteBufferUtil.copyBytes(record, record.position(), out, out.position(), recordSize);
 
-        out.writeInt((int) crc.getValue());
+        // update and write crcs
+        CRC32 crc = Crc.crc32();
+        out.position(start);
+        out.limit(fixedCrcPosition);
+        crc.update(out);
+        out.limit(recordEnd);
+        out.putInt((int) crc.getValue());
+        crc.update(out);
+        out.limit(recordEnd + 4);
+        out.putInt((int) crc.getValue());
     }
 
+    // we reuse record as the value we return
     static <K> void read(EntryHolder<K> into,
                          KeySupport<K> keySupport,
                          ByteBuffer from,
                          int userVersion)
     throws IOException
     {
-        CRC32 crc = Crc.crc32();
         into.clear();
 
-        try (DataInputBuffer in = new DataInputBuffer(from, false))
+        int start = from.position();
         {
-            K key = keySupport.deserialize(in, userVersion);
-            keySupport.updateChecksum(crc, key, userVersion);
-            into.key = key;
+            int totalSize = from.getInt(start) - start;
+            Invariants.checkState(totalSize == from.remaining());
 
-            int hostCount = in.readShort();
-            updateChecksumShort(crc, (short) hostCount);
+            CRC32 crc = Crc.crc32();
+            int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
+            int fixedCrc = readAndUpdateFixedCrc(crc, from, fixedSize);
+            validateCRC(crc, fixedCrc);
 
-            int entrySize = in.readInt();
-            updateChecksumInt(crc, entrySize);
-
-            validateCRC(crc, in.readInt());
-
-            for (int i = 0; i < hostCount; i++)
-            {
-                int hostId = in.readInt();
-                updateChecksumInt(crc, hostId);
-                into.hosts.add(hostId);
-            }
-
-            // TODO: try to avoid allocating another buffer here
-            ByteBuffer entry = ByteBufferUtil.read(in, entrySize);
-            updateChecksum(crc, entry);
-            into.value = entry;
-            into.userVersion = userVersion;
-
-            validateCRC(crc, in.readInt());
+            int recordCrc = readAndUpdateRecordCrc(crc, from, start + totalSize);
+            validateCRC(crc, recordCrc);
         }
+
+        readValidated(into, from, start, keySupport, userVersion);
     }
 
-    static <K> boolean tryRead(EntryHolder<K> into,
-                               KeySupport<K> keySupport,
-                               ByteBuffer from,
-                               DataInputBuffer in,
-                               int syncedOffset,
-                               int userVersion)
+    // slices the provided buffer to assign to into.value
+    static <K> int tryRead(EntryHolder<K> into,
+                           KeySupport<K> keySupport,
+                           ByteBuffer from,
+                           int syncedOffset,
+                           int userVersion)
     throws IOException
     {
         CRC32 crc = Crc.crc32();
         into.clear();
 
-        int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
-        if (from.remaining() < fixedSize)
+        int start = from.position();
+        if (from.remaining() < TypeSizes.INT_SIZE)
+            return -1;
+
+        int totalSize = from.getInt(start) - start;
+        if (totalSize == 0)
+            return -1;
+
+        if (from.remaining() < totalSize)
             return handleReadException(new EOFException(), from.limit(), syncedOffset);
 
-        updateChecksum(crc, from, from.position(), fixedSize - TypeSizes.INT_SIZE);
-        int fixedCrc = from.getInt(from.position() + fixedSize - TypeSizes.INT_SIZE);
+        {
+            int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
+            int fixedCrc = readAndUpdateFixedCrc(crc, from, fixedSize);
+            try
+            {
+                validateCRC(crc, fixedCrc);
+            }
+            catch (IOException e)
+            {
+                return handleReadException(e, from.position() + fixedSize, syncedOffset);
+            }
 
-        try
-        {
-            validateCRC(crc, fixedCrc);
-        }
-        catch (IOException e)
-        {
-            return handleReadException(e, from.position() + fixedSize, syncedOffset);
+            int recordCrc = readAndUpdateRecordCrc(crc, from, start + totalSize);
+            try
+            {
+                validateCRC(crc, recordCrc);
+            }
+            catch (IOException e)
+            {
+                return handleReadException(e, from.position(), syncedOffset);
+            }
         }
 
-        int hostCount, recordSize;
-        try
-        {
-            into.key = keySupport.deserialize(in, userVersion);
-            hostCount = in.readShort();
-            recordSize = in.readInt();
-            in.skipBytesFully(TypeSizes.INT_SIZE);
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(); // can't happen unless deserializer is buggy
-        }
-
-        int variableSize = EntrySerializer.variableEntrySize(hostCount, recordSize);
-        if (from.remaining() < variableSize)
-            return handleReadException(new EOFException(), from.limit(), syncedOffset);
-
-        updateChecksum(crc, from, from.position(), variableSize - TypeSizes.INT_SIZE);
-        int variableCrc = from.getInt(from.position() + variableSize - TypeSizes.INT_SIZE);
-
-        try
-        {
-            validateCRC(crc, variableCrc);
-        }
-        catch (IOException e)
-        {
-            return handleReadException(e, from.position() + variableSize, syncedOffset);
-        }
-
-        for (int i = 0; i < hostCount; i++)
-        {
-            into.hosts.add(in.readInt());
-        }
-
-        try
-        {
-            in.skipBytesFully(recordSize);
-        }
-        catch (IOException e)
-        {
-            throw new AssertionError(); // can't happen
-        }
-
-        into.value = from.duplicate()
-                         .position(from.position() - recordSize)
-                         .limit(from.position());
-        into.userVersion = userVersion;
-
-        in.skipBytesFully(TypeSizes.INT_SIZE);
-        return true;
+        readValidated(into, from, start, keySupport, userVersion);
+        return totalSize;
     }
 
-    private static boolean handleReadException(IOException e, int bufferPosition, int fsyncedLimit) throws IOException
+    private static <K> void readValidated(EntryHolder<K> into, ByteBuffer from, int start, KeySupport<K> keySupport, int userVersion)
+    {
+        from.position(start + TypeSizes.INT_SIZE);
+        into.key = keySupport.deserialize(from, userVersion);
+        int hostCount = from.getShort();
+
+        from.position(from.position() + 4);
+        for (int i = 0; i < hostCount; i++)
+        {
+            int hostId = from.getInt();
+            into.hosts.add(hostId);
+        }
+
+        into.value = from;
+        into.userVersion = userVersion;
+    }
+
+    private static int readAndUpdateFixedCrc(CRC32 crc, ByteBuffer from, int fixedSize)
+    {
+        int fixedEnd = from.position() + fixedSize - TypeSizes.INT_SIZE;
+        int fixedCrc = from.getInt(fixedEnd);
+        from.limit(fixedEnd);
+        crc.update(from);
+        return fixedCrc;
+    }
+
+    private static int readAndUpdateRecordCrc(CRC32 crc, ByteBuffer from, int limit)
+    {
+        int recordEnd = limit - TypeSizes.INT_SIZE;
+        from.limit(limit);
+        int recordCrc = from.getInt(recordEnd);
+        from.position(from.position() + 4);
+        from.limit(recordEnd);
+        crc.update(from);
+        return recordCrc;
+    }
+
+    private static int handleReadException(IOException e, int bufferPosition, int fsyncedLimit) throws IOException
     {
         if (bufferPosition <= fsyncedLimit)
             throw e;
         else
-            return false;
+            return -1;
     }
 
     static <K> int fixedEntrySize(KeySupport<K> keySupport, int userVersion)
     {
         return keySupport.serializedSize(userVersion) // key/id
              + TypeSizes.SHORT_SIZE                   // host count
-             + TypeSizes.INT_SIZE                     // record size
+             + TypeSizes.INT_SIZE                     // total size
              + TypeSizes.INT_SIZE;                    // CRC
     }
 
-    static int variableEntrySize(int hostCount, int recordSize)
+    static int variableEntrySize(int hostCount)
     {
         return TypeSizes.INT_SIZE * hostCount // hosts
-             + recordSize                     // record
              + TypeSizes.INT_SIZE;            // CRC
     }
 

src/java/org/apache/cassandra/journal/Flusher.java

@@ -45,7 +45,6 @@ import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe
 import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
 import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN;
 import static org.apache.cassandra.journal.Params.FlushMode.PERIODIC;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized;
 import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
 import static org.apache.cassandra.utils.Simulate.With.GLOBAL_CLOCK;
@@ -69,35 +68,36 @@ final class Flusher<K, V>
     private final AtomicLong written = new AtomicLong(0);
 
     // the time of the last initiated flush
-    volatile long flushStartedAt = nanoTime();
+    volatile long flushStartedAt;
     // the time of the earliest flush that has completed an fsync; all Allocations written before this time are durable
     volatile long fsyncFinishedFor = flushStartedAt;
+    volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0);
 
     // a signal that writers can wait on to be notified of a completed flush in PERIODIC FlushMode
     private final WaitQueue fsyncComplete = newWaitQueue(); // TODO (expected): this is only used for testing, can we remove this?
+    private final MonotonicClock clock = preciseTime;
 
     // a signal and flag that callers outside the flusher thread can use
     // to signal they want the journal segments to be flushed to disk
     private final Semaphore haveWork = newSemaphore(1);
     private volatile boolean flushRequested;
 
-    private final FlushMethod<K, V> syncFlushMethod;
-    private final FlushMethod<K, V> asyncFlushMethod;
+    private final Mode<K, V> mode;
     private final Callbacks callbacks;
 
     Flusher(Journal<K, V> journal, Callbacks callbacks)
     {
         this.journal = journal;
         this.params = journal.params;
-        this.syncFlushMethod = syncFlushMethod(params);
-        this.asyncFlushMethod = asyncFlushMethod(params);
+        this.mode = mode(params);
         this.callbacks = callbacks;
     }
 
     void start()
     {
         String flushExecutorName = journal.name + "-disk-flusher-" + toLowerCaseLocalized(params.flushMode().toString());
-        flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new FlushRunnable(preciseTime), SAFE, NON_DAEMON, SYNCHRONIZED);
+        flushStartedAt = clock.now();
+        flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new FlushRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
     }
 
     void shutdown() throws InterruptedException
@@ -112,6 +112,7 @@ final class Flusher<K, V>
     }
 
     @Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
+    // waits for writes to complete before triggering an fsync
     private class FlushRunnable implements Interruptible.Task
     {
         @Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
@@ -182,6 +183,7 @@ final class Flusher<K, V>
                 fsyncStartedFor = startedAt;
                 // synchronized to prevent thread interrupts while performing IO operations and also
                 // clear interrupted status to prevent ClosedByInterruptException in ActiveSegment::flush
+                int fsyncedTo;
                 synchronized (this)
                 {
                     boolean ignore = Thread.interrupted();
@@ -191,17 +193,19 @@ final class Flusher<K, V>
                         journal.closeActiveSegmentAndOpenAsStatic(fsyncing);
                         fsyncing = journal.getActiveSegment(fsyncing.descriptor.timestamp + 1);
                     }
-                    fsyncing.fsync();
+                    fsyncedTo = fsyncTo.writtenToAtLeast();
+                    fsyncTo.fsync();
                 }
+                fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, startedAt);
                 fsyncFinishedFor = startedAt;
                 fsyncComplete.signalAll();
                 long finishedAt = clock.now();
                 processDuration(startedAt, finishedAt);
             }
 
-            void afterFlush(long startedAt, ActiveSegment<K, V> segment, int syncedOffset)
+            void afterFlush(long startedAt, ActiveSegment<K, V> segment)
             {
-                long requireFsyncTo = startedAt - periodicFlushLagBlockNanos();
+                long requireFsyncTo = startedAt - periodicBlockNanos();
 
                 fsyncUpTo = segment;
                 fsyncWaitingSince = startedAt;
@@ -210,26 +214,11 @@ final class Flusher<K, V>
 
                 if (requireFsyncTo > fsyncFinishedFor)
                     awaitFsyncAt(requireFsyncTo, journal.metrics.waitingOnFlush.time());
-                callbacks.onFlush(segment.descriptor.timestamp, syncedOffset);
-            }
-
-            private void doNoOpFlush(long startedAt)
-            {
-                if (fsyncFinishedFor >= fsyncWaitingSince)
-                {
-                    fsyncFinishedFor = startedAt;
-                }
-                else
-                {
-                    // if the flusher is still running, update the waitingSince register
-                    fsyncWaitingSince = startedAt;
-                    notify(awaitingWork);
-                }
+                callbacks.onFsync();
             }
         }
 
         private final NoSpamLogger noSpamLogger;
-        private final MonotonicClock clock;
         private final @Nullable FSyncRunnable fSyncRunnable;
 
         private ActiveSegment<K, V> current = null;
@@ -240,10 +229,9 @@ final class Flusher<K, V>
         private long duration = 0;              // time spent flushing since firstLaggedAt
         private long lagDuration = 0;                // cumulative lag since firstLaggedAt
 
-        FlushRunnable(MonotonicClock clock)
+        FlushRunnable()
         {
             this.noSpamLogger = NoSpamLogger.wrap(logger, 5, MINUTES);
-            this.clock = clock;
             this.fSyncRunnable = params.flushMode() == PERIODIC ? newFsyncRunnable() : null;
         }
 
@@ -287,6 +275,7 @@ final class Flusher<K, V>
 
             if (flushPeriodNanos <= 0)
             {
+                Invariants.checkState(params.flushMode() != PERIODIC);
                 haveWork.acquire(1);
             }
             else
@@ -296,7 +285,7 @@ final class Flusher<K, V>
             }
         }
 
-        private void doFlush(long startedAt) throws InterruptedException
+        private void doFlush(long startedAt)
         {
             boolean synchronousFsync = fSyncRunnable == null;
 
@@ -304,32 +293,33 @@ final class Flusher<K, V>
                 current = journal.oldestActiveSegment();
             ActiveSegment<K, V> newCurrent = journal.currentActiveSegment();
 
-            if (newCurrent == current && (newCurrent == null || !newCurrent.shouldFlush()))
-            {
-                if (synchronousFsync) fsyncFinishedFor = startedAt;
-                else fSyncRunnable.doNoOpFlush(startedAt);
-
-                if (current != null)
-                    callbacks.onFlush(current.descriptor.timestamp, (int) current.lastFlushedOffset());
+            if (newCurrent == null)
                 return;
-            }
-
-            Invariants.checkState(newCurrent != null);
 
             try
             {
                 while (current != newCurrent)
                 {
                     current.discardUnusedTail();
-                    current.flush(synchronousFsync);
+                    current.updateWrittenTo();
                     if (synchronousFsync)
+                    {
+                        current.fsync();
                         journal.closeActiveSegmentAndOpenAsStatic(current);
+                    }
                     current = journal.getActiveSegment(current.descriptor.timestamp + 1);
                 }
-                int syncedOffset = current.flush(synchronousFsync);
 
-                if (synchronousFsync) afterFSync(startedAt, current.descriptor.timestamp, syncedOffset);
-                else fSyncRunnable.afterFlush(startedAt, current, syncedOffset);
+                int writtenTo = current.updateWrittenTo();
+                if (synchronousFsync)
+                {
+                    current.fsync();
+                    afterFSync(startedAt, current.descriptor.timestamp, writtenTo);
+                }
+                else
+                {
+                    fSyncRunnable.afterFlush(startedAt, current);
+                }
             }
             catch (Throwable t)
             {
@@ -373,10 +363,11 @@ final class Flusher<K, V>
             }
         }
 
-        private void afterFSync(long startedAt, long syncedSegment, int syncedOffset)
+        private void afterFSync(long startedAt, long segment, int position)
         {
+            fsyncFinishedForPosition = new RecordPointer(segment, position, startedAt);
             fsyncFinishedFor = startedAt;
-            callbacks.onFlush(syncedSegment, syncedOffset);
+            callbacks.onFsync();
             fsyncComplete.signalAll();
             long finishedAt = clock.now();
             processDuration(startedAt, finishedAt);
@@ -390,88 +381,120 @@ final class Flusher<K, V>
         }
     }
 
-    @FunctionalInterface
-    private interface FlushMethod<K, V>
+    private interface Mode<K, V>
     {
-        void flush(ActiveSegment<K, V>.Allocation allocation);
+        void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc);
+        RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc);
+        boolean isDurable(RecordPointer recordPointer);
     }
 
-    private FlushMethod<K, V> syncFlushMethod(Params params)
+    private class BatchMode implements Mode<K, V>
     {
-        switch (params.flushMode())
-        {
-            default: throw new IllegalArgumentException();
-            case    BATCH: return this::waitForFlushBatch;
-            case    GROUP: return this::waitForFlushGroup;
-            case PERIODIC: return this::waitForFlushPeriodic;
-        }
-    }
-
-    private FlushMethod<K, V> asyncFlushMethod(Params params)
-    {
-        switch (params.flushMode())
-        {
-            default: throw new IllegalArgumentException();
-            case    BATCH: return this::asyncFlushBatch;
-            case    GROUP: return this::asyncFlushGroup;
-            case PERIODIC: return this::asyncFlushPeriodic;
-        }
-    }
-
-    void waitForFlush(ActiveSegment<K, V>.Allocation alloc)
-    {
-        syncFlushMethod.flush(alloc);
-    }
-
-    void asyncFlush(ActiveSegment<K, V>.Allocation alloc)
-    {
-        asyncFlushMethod.flush(alloc);
-    }
-
-    private void waitForFlushBatch(ActiveSegment<K, V>.Allocation alloc)
-    {
-        pending.incrementAndGet();
-        requestExtraFlush();
-        alloc.awaitFlush(journal.metrics.waitingOnFlush);
-        pending.decrementAndGet();
-        written.incrementAndGet();
-    }
-
-    private void waitForFlushGroup(ActiveSegment<K, V>.Allocation alloc)
-    {
-        pending.incrementAndGet();
-        alloc.awaitFlush(journal.metrics.waitingOnFlush);
-        pending.decrementAndGet();
-        written.incrementAndGet();
-    }
-
-    private void waitForFlushPeriodic(ActiveSegment<K, V>.Allocation ignore)
-    {
-        long expectedFlushTime = nanoTime() - periodicFlushLagBlockNanos();
-        if (fsyncFinishedFor < expectedFlushTime)
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
         {
             pending.incrementAndGet();
-            awaitFsyncAt(expectedFlushTime, journal.metrics.waitingOnFlush.time());
+            requestExtraFlush();
+            alloc.awaitDurable(journal.metrics.waitingOnFlush);
             pending.decrementAndGet();
+            written.incrementAndGet();
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            requestExtraFlush();
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer pointer)
+        {
+            return pointer.compareTo(fsyncFinishedForPosition) <= 0;
         }
-        written.incrementAndGet();
     }
 
-    private void asyncFlushBatch(ActiveSegment<K, V>.Allocation alloc)
+    private class GroupMode implements Mode<K, V>
     {
-        requestExtraFlush();
-        written.incrementAndGet();
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+        {
+            pending.incrementAndGet();
+            alloc.awaitDurable(journal.metrics.waitingOnFlush);
+            pending.decrementAndGet();
+            written.incrementAndGet();
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer pointer)
+        {
+            return pointer.compareTo(fsyncFinishedForPosition) <= 0;
+        }
     }
 
-    private void asyncFlushGroup(ActiveSegment<K, V>.Allocation alloc)
+    private class PeriodicMode implements Mode<K, V>
     {
-        written.incrementAndGet();
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+        {
+            RecordPointer pointer = flushAsync(alloc);
+
+            long expectedFsyncTime = pointer.writtenAt - periodicBlockNanos();
+            if (expectedFsyncTime > fsyncFinishedFor)
+            {
+                pending.incrementAndGet();
+                awaitFsyncAt(expectedFsyncTime, journal.metrics.waitingOnFlush.time());
+                pending.decrementAndGet();
+            }
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), clock.now());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer alloc)
+        {
+            long expectedFsyncTime = alloc.writtenAt - periodicBlockNanos();
+            return expectedFsyncTime <= fsyncFinishedFor;
+        }
     }
 
-    private void asyncFlushPeriodic(ActiveSegment<K, V>.Allocation ignore)
+    Mode<K, V> mode(Params params)
     {
-        requestExtraFlush();
-        written.incrementAndGet();
+        switch (params.flushMode())
+        {
+            default: throw new AssertionError("Unexpected FlushMode: " + params.flushMode());
+            case BATCH: return new BatchMode();
+            case GROUP: return new GroupMode();
+            case PERIODIC: return new PeriodicMode();
+        }
+    }
+
+    RecordPointer flush(ActiveSegment<K, V>.Allocation alloc)
+    {
+        return mode.flushAsync(alloc);
+    }
+
+    void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+    {
+        mode.flushAndAwaitDurable(alloc);
+    }
+
+    boolean isDurable(RecordPointer pointer)
+    {
+        return mode.isDurable(pointer);
     }
 
     /**
@@ -505,12 +528,12 @@ final class Flusher<K, V>
 
     private long flushPeriodNanos()
     {
-        return 1_000_000L * params.flushPeriodMillis();
+        return params.flushPeriod(NANOSECONDS);
     }
 
-    private long periodicFlushLagBlockNanos()
+    private long periodicBlockNanos()
     {
-        return 1_000_000L * params.periodicFlushLagBlock();
+        return params.periodicBlockPeriod(NANOSECONDS);
     }
 
     long pendingEntries()
@@ -531,8 +554,9 @@ final class Flusher<K, V>
          * completed and also flushed.
          * callbacks for all entries earlier than (segment, position) have finished execution.
          */
-        void onFlush(long segment, int position);
+        void onFsync();
 
+        // TODO (required): tie this to specific allocations..
         void onFlushFailed(Throwable cause);
     }
 }

src/java/org/apache/cassandra/journal/InMemoryIndex.java

@@ -102,7 +102,10 @@ final class InMemoryIndex<K> extends Index<K>
     @Override
     public long[] lookUp(K id)
     {
-        return mayContainId(id) ? index.getOrDefault(id, EMPTY) : EMPTY;
+        K lastId = lastId();
+        if (lastId == null || keySupport.compare(id, lastId) <= 0)
+            return index.getOrDefault(id, EMPTY);
+        return EMPTY;
     }
 
     @Override

src/java/org/apache/cassandra/journal/Index.java

@@ -21,8 +21,6 @@ import javax.annotation.Nullable;
 
 import org.apache.cassandra.utils.Closeable;
 
-import static com.google.common.collect.Iterables.any;
-
 /**
  * Mapping of client supplied ids to in-segment offsets
  */
@@ -75,15 +73,7 @@ abstract class Index<K> implements Closeable
         K firstId = firstId();
         K lastId = lastId();
 
-        return null != firstId && null != lastId && keySupport.compare(id, firstId) >= 0 && keySupport.compare(id, lastId) <= 0;
-    }
-
-    /**
-     * @return whether any of the ids falls within lower/upper bounds of the index
-     */
-    boolean mayContainIds(Iterable<K> ids)
-    {
-        return any(ids, this::mayContainId);
+        return null != firstId && keySupport.compare(id, firstId) >= 0 && (null == lastId || keySupport.compare(id, lastId) <= 0);
     }
 
     /**

src/java/org/apache/cassandra/journal/Journal.java

@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
 import com.codahale.metrics.Timer.Context;
-import org.agrona.collections.ObjectHashSet;
 import org.apache.cassandra.concurrent.Interruptible;
 import org.apache.cassandra.concurrent.Interruptible.TerminateException;
 import org.apache.cassandra.concurrent.SequentialExecutorPlus;
@@ -51,12 +50,12 @@ import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.PathUtils;
-import org.apache.cassandra.journal.Segments.ReferencedSegment;
 import org.apache.cassandra.journal.Segments.ReferencedSegments;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Crc;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Simulate;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
 import org.jctools.queues.MpscUnboundedArrayQueue;
 
@@ -119,8 +118,9 @@ public class Journal<K, V> implements Shutdownable
     private final WaitQueue allocatorThreadWaitQueue = newWaitQueue();
     private final BooleanSupplier allocatorThreadWaitCondition = () -> (availableSegment == null);
     private final FlusherCallbacks flusherCallbacks;
+    final OpOrder readOrder = new OpOrder();
 
-    SequentialExecutorPlus closer;
+    SequentialExecutorPlus closer, releaser;
 
     private class FlusherCallbacks implements Flusher.Callbacks
     {
@@ -128,17 +128,14 @@ public class Journal<K, V> implements Shutdownable
         private List<WaitingFor> drained = new ArrayList<>();
 
         @Override
-        public void onFlush(long segment, int position)
+        public void onFsync()
         {
-            // TODO (required): this seems to be a big source of allocations
             waitingFor.drain(drained::add);
             List<WaitingFor> remaining = new ArrayList<>();
             for (WaitingFor wait : drained)
             {
-                if (wait.segment == segment && wait.position <= position)
-                    wait.run();
-                else
-                    remaining.add(wait);
+                if (flusher.isDurable(wait)) wait.run();
+                else remaining.add(wait);
             }
             drained = remaining;
         }
@@ -151,13 +148,10 @@ public class Journal<K, V> implements Shutdownable
 
         private void submit(RecordPointer pointer, Runnable runnable)
         {
-            if (isFlushed(pointer))
+            if (flusher.isDurable(pointer))
                 runnable.run();
             else
-            {
-                waitingFor.add(new WaitingFor(pointer.segment, pointer.position, runnable));
-                flusher.requestExtraFlush();
-            }
+                waitingFor.add(new WaitingFor(pointer, runnable));
         }
     }
 
@@ -165,9 +159,9 @@ public class Journal<K, V> implements Shutdownable
     {
         private final Runnable onFlush;
 
-        public WaitingFor(long segment, int position, Runnable onFlush)
+        public WaitingFor(RecordPointer pointer, Runnable onFlush)
         {
-            super(segment, position);
+            super(pointer);
             this.onFlush = onFlush;
         }
 
@@ -197,19 +191,9 @@ public class Journal<K, V> implements Shutdownable
         this.compactor = new Compactor<>(this, segmentCompactor);
     }
 
-    public boolean isFlushed(RecordPointer recordPointer)
+    public void onDurable(RecordPointer recordPointer, Runnable runnable)
     {
-        Segment<K, V> current = currentSegment;
-        if (current.descriptor.timestamp == recordPointer.segment)
-            return current.isFlushed(recordPointer.position);
-
-        return segments.get().isFlushed(recordPointer);
-    }
-
-    public void onFlush(RecordPointer recordPointer, Runnable runnable)
-    {
-        if (isFlushed(recordPointer)) runnable.run();
-        else flusherCallbacks.submit(recordPointer, runnable);
+        flusherCallbacks.submit(recordPointer, runnable);
     }
 
     public void start()
@@ -230,12 +214,13 @@ public class Journal<K, V> implements Shutdownable
 
         segments.set(Segments.of(StaticSegment.open(descriptors, keySupport)));
         closer = executorFactory().sequential(name + "-closer");
+        releaser = executorFactory().sequential(name + "-releaser");
         allocator = executorFactory().infiniteLoop(name + "-allocator", new AllocateRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
         advanceSegment(null);
-        flusher.start();
-        compactor.start();
         Invariants.checkState(state.compareAndSet(State.INITIALIZING, State.NORMAL),
                               "Unexpected journal state after initialization", state);
+        flusher.start();
+        compactor.start();
     }
 
     @VisibleForTesting
@@ -272,13 +257,16 @@ public class Journal<K, V> implements Shutdownable
                                   "Unexpected journal state while trying to shut down", state);
             allocator.shutdown();
             wakeAllocator(); // Wake allocator to force it into shutdown
+            // TODO (expected): why are we awaitingTermination here when we have a separate method for it?
             allocator.awaitTermination(1, TimeUnit.MINUTES);
             segmentPrepared.signalAll(); // Wake up all threads waiting on the new segment
             compactor.shutdown();
             compactor.awaitTermination(1, TimeUnit.MINUTES);
             flusher.shutdown();
             closer.shutdown();
+            releaser.shutdown();
             closer.awaitTermination(1, TimeUnit.MINUTES);
+            releaser.awaitTermination(1, TimeUnit.MINUTES);
             closeAllSegments();
             metrics.deregister();
             Invariants.checkState(state.compareAndSet(State.SHUTDOWN, State.TERMINATED),
@@ -303,6 +291,7 @@ public class Journal<K, V> implements Shutdownable
         boolean r = true;
         r &= allocator.awaitTermination(timeout, units);
         r &= closer.awaitTermination(timeout, units);
+        r &= releaser.awaitTermination(timeout, units);
         return r;
     }
 
@@ -323,9 +312,9 @@ public class Journal<K, V> implements Shutdownable
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
 
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.allSorted(true))
+            for (Segment<K, V> segment : segments.get().allSorted(true))
             {
                 if (segment.readLast(id, holder))
                 {
@@ -347,9 +336,9 @@ public class Journal<K, V> implements Shutdownable
     public void readAll(K id, RecordConsumer<K> consumer)
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.allSorted(false))
+            for (Segment<K, V> segment : segments.get().allSorted(false))
                 segment.readAll(id, holder, consumer);
         }
     }
@@ -390,9 +379,9 @@ public class Journal<K, V> implements Shutdownable
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
 
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.all())
+            for (Segment<K, V> segment : segments.get().all())
             {
                 long[] offsets = segment.index().lookUp(id);
                 for (long offsetAndSize : offsets)
@@ -436,40 +425,18 @@ public class Journal<K, V> implements Shutdownable
     @SuppressWarnings("unused")
     public boolean readLast(K id, RecordConsumer<K> consumer)
     {
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.all())
+            for (Segment<K, V> segment : segments.get().allSorted(false))
+            {
+                if (!segment.index().mayContainId(id))
+                    continue;
+
                 if (segment.readLast(id, consumer))
                     return true;
-        }
-        return false;
-    }
-
-    /**
-     * Test for existence of entries with specified ids.
-     *
-     * @return subset of ids to test that have been found in the journal
-     */
-    @SuppressWarnings("unused")
-    public Set<K> test(Set<K> test)
-    {
-        Set<K> present = new ObjectHashSet<>(test.size() + 1, 0.9f);
-        try (ReferencedSegments<K, V> segments = selectAndReference(test))
-        {
-            for (Segment<K, V> segment : segments.all())
-            {
-                for (K id : test)
-                {
-                    if (segment.index().lookUpLast(id) != -1)
-                    {
-                        present.add(id);
-                        if (test.size() == present.size())
-                            return present;
-                    }
-                }
             }
         }
-        return present;
+        return false;
     }
 
     /**
@@ -488,7 +455,7 @@ public class Journal<K, V> implements Shutdownable
             valueSerializer.serialize(id, record, dob, params.userVersion());
             ActiveSegment<K, V>.Allocation alloc = allocate(dob.getLength(), hosts);
             alloc.writeInternal(id, dob.unsafeGetBufferAndFlip(), hosts);
-            flusher.waitForFlush(alloc);
+            flusher.flushAndAwaitDurable(alloc);
         }
         catch (IOException e)
         {
@@ -514,20 +481,18 @@ public class Journal<K, V> implements Shutdownable
 
     public RecordPointer asyncWrite(K id, Writer writer, Set<Integer> hosts)
     {
-        RecordPointer recordPointer;
         try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
         {
             writer.write(dob, params.userVersion());
             ActiveSegment<K, V>.Allocation alloc = allocate(dob.getLength(), hosts);
-            recordPointer = alloc.write(id, dob.unsafeGetBufferAndFlip(), hosts);
-            flusher.asyncFlush(alloc);
+            alloc.write(id, dob.unsafeGetBufferAndFlip(), hosts);
+            return flusher.flush(alloc);
         }
         catch (IOException e)
         {
             // exception during record serialization into the scratch buffer
             throw new RuntimeException(e);
         }
-        return recordPointer;
     }
 
     private ActiveSegment<K, V>.Allocation allocate(int entrySize, Set<Integer> hosts)
@@ -615,7 +580,7 @@ public class Journal<K, V> implements Shutdownable
             availableSegment = null;
         }
         if (next != null)
-            next.closeAndDiscard();
+            next.closeAndDiscard(this);
     }
 
     private class AllocateRunnable implements Interruptible.Task
@@ -715,52 +680,9 @@ public class Journal<K, V> implements Shutdownable
         for (Segment<K, V> segment : segments.all())
         {
             if (segment.isActive())
-                ((ActiveSegment<K, V>) segment).closeAndIfEmptyDiscard();
+                ((ActiveSegment<K, V>) segment).closeAndIfEmptyDiscard(this);
             else
-                segment.close();
-        }
-    }
-
-    /**
-     * Select segments that could potentially have any entry with the specified id and
-     * attempt to grab references to them all.
-     *
-     * @return a subset of segments with references to them
-     */
-    ReferencedSegments<K, V> selectAndReference(K id)
-    {
-        while (true)
-        {
-            ReferencedSegments<K, V> referenced = segments().selectAndReference(s -> s.index().mayContainId(id));
-            if (null != referenced)
-                return referenced;
-        }
-    }
-
-    /**
-     * Select segments that could potentially have any entry with the specified ids and
-     * attempt to grab references to them all.
-     *
-     * @return a subset of segments with references to them
-     */
-    ReferencedSegments<K, V> selectAndReference(Iterable<K> ids)
-    {
-        while (true)
-        {
-            ReferencedSegments<K, V> referenced = segments().selectAndReference(s -> s.index().mayContainIds(ids));
-            if (null != referenced)
-                return referenced;
-        }
-    }
-
-    @SuppressWarnings("unused")
-    ReferencedSegment<K, V> selectAndReference(long segmentTimestamp)
-    {
-        while (true)
-        {
-            ReferencedSegment<K, V> referenced = segments().selectAndReference(segmentTimestamp);
-            if (null != referenced)
-                return referenced;
+                segment.close(this);
         }
     }
 
@@ -886,10 +808,11 @@ public class Journal<K, V> implements Shutdownable
         public void run()
         {
             activeSegment.discardUnusedTail();
-            activeSegment.flush(true);
+            activeSegment.updateWrittenTo();
+            activeSegment.fsync();
             activeSegment.persistComponents();
             replaceCompletedSegment(activeSegment, StaticSegment.open(activeSegment.descriptor, keySupport));
-            activeSegment.release();
+            activeSegment.release(Journal.this);
         }
     }
 
@@ -898,7 +821,7 @@ public class Journal<K, V> implements Shutdownable
         if (activeSegment.isEmpty())
         {
             removeEmptySegment(activeSegment);
-            activeSegment.closeAndDiscard();
+            activeSegment.closeAndDiscard(this);
             return;
         }
 
@@ -1000,6 +923,7 @@ public class Journal<K, V> implements Shutdownable
      */
     public class StaticSegmentIterator implements Closeable
     {
+        // TODO (expected): use MergeIterator
         private final PriorityQueue<StaticSegment.KeyOrderReader<K>> readers;
         private final ReferencedSegments<K, V> segments;
 

src/java/org/apache/cassandra/journal/KeySupport.java

@@ -36,10 +36,12 @@ public interface KeySupport<K> extends Comparator<K>
     int serializedSize(int userVersion);
 
     void serialize(K key, DataOutputPlus out, int userVersion) throws IOException;
+    void serialize(K key, ByteBuffer out, int userVersion) throws IOException;
 
     K deserialize(DataInputPlus in, int userVersion) throws IOException;
 
     K deserialize(ByteBuffer buffer, int position, int userVersion);
+    K deserialize(ByteBuffer buffer, int userVersion);
 
     void updateChecksum(Checksum crc, K key, int userVersion);
 

src/java/org/apache/cassandra/journal/Metadata.java

@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.journal;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
@@ -45,6 +46,7 @@ final class Metadata
     private final Set<Integer> unmodifiableHosts;
     private final Map<Integer, Integer> recordsPerHost;
 
+    private int fsyncLimit;
     private volatile int recordsCount;
     private static final AtomicIntegerFieldUpdater<Metadata> recordsCountUpdater =
         AtomicIntegerFieldUpdater.newUpdater(Metadata.class, "recordsCount");
@@ -73,6 +75,11 @@ final class Metadata
             recordsPerHost.compute(host, (k, v) -> null == v ? 1 : v + 1);
     }
 
+    int fsyncLimit()
+    {
+        return fsyncLimit;
+    }
+
     private void incrementRecordsCount()
     {
         recordsCountUpdater.incrementAndGet(this);
@@ -186,12 +193,12 @@ final class Metadata
         }
     }
 
-    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport)
     {
         Int2IntHashMap recordsPerHost = new Int2IntHashMap(Integer.MIN_VALUE);
         int recordsCount = 0;
 
-        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, fsyncedLimit))
+        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, Integer.MAX_VALUE))
         {
             while (reader.advance())
             {
@@ -203,13 +210,19 @@ final class Metadata
                 ++recordsCount;
             }
         }
+        catch (JournalReadError e)
+        {
+            // we expect EOF when rebuilding
+            if (!(e.getCause() instanceof EOFException))
+                throw e;
+        }
 
         return new Metadata(recordsPerHost, recordsCount);
     }
 
-    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport)
     {
-        Metadata metadata = rebuild(descriptor, keySupport, fsyncedLimit);
+        Metadata metadata = rebuild(descriptor, keySupport);
         metadata.persist(descriptor);
         return metadata;
     }

src/java/org/apache/cassandra/journal/Params.java

@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.concurrent.TimeUnit;
+
 public interface Params
 {
     enum FlushMode { BATCH, GROUP, PERIODIC }
@@ -40,23 +42,18 @@ public interface Params
 
     boolean enableCompaction();
 
-    int compactionPeriodMillis();
+    long compactionPeriod(TimeUnit units);
 
     /**
      * @return milliseconds between journal flushes
      */
-    int flushPeriodMillis();
-
-    default int flushPeriodNanos()
-    {
-        return flushPeriodMillis() * 1_000_000;
-    }
+    long flushPeriod(TimeUnit units);
 
     /**
-     * @return milliseconds to block writes for while waiting for a slow disk flush to complete
-     *         when in {@link FlushMode#PERIODIC} mode
+     * @return to block writes for while waiting for a slow fsync to complete
+     * when in {@link FlushMode#PERIODIC} mode
      */
-    int periodicFlushLagBlock();
+    long periodicBlockPeriod(TimeUnit units);
 
     /**
      * @return user provided version to use for key and value serialization

src/java/org/apache/cassandra/journal/RecordPointer.java

@@ -26,11 +26,23 @@ public class RecordPointer implements Comparable<RecordPointer>
 {
     public final long segment; // unique segment id
     public final int position; // record start position within the segment
+    public final long writtenAt; // only set for periodic mode
 
     public RecordPointer(long segment, int position)
+    {
+        this(segment, position, 0);
+    }
+
+    public RecordPointer(long segment, int position, long writtenAt)
     {
         this.segment = segment;
         this.position = position;
+        this.writtenAt = writtenAt;
+    }
+
+    public RecordPointer(RecordPointer pointer)
+    {
+        this(pointer.segment, pointer.position, pointer.writtenAt);
     }
 
     @Override

src/java/org/apache/cassandra/journal/Segment.java

@@ -20,25 +20,44 @@ package org.apache.cassandra.journal;
 import java.nio.ByteBuffer;
 
 import accord.utils.Invariants;
+import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.SelfRefCounted;
 
-public abstract class Segment<K, V> implements Closeable, RefCounted<Segment<K, V>>
+public abstract class Segment<K, V> implements SelfRefCounted<Segment<K, V>>, Comparable<Segment<K, V>>
 {
+    protected abstract static class Tidier implements Tidy, Runnable
+    {
+        OpOrder.Barrier await;
+        ExecutorPlus executor;
+
+        abstract void onUnreferenced();
+
+        public final void run()
+        {
+            await.await();
+            onUnreferenced();
+        }
+
+        public final void tidy()
+        {
+            executor.execute(this);
+        }
+    }
+
     final File file;
     final Descriptor descriptor;
-    final SyncedOffsets syncedOffsets;
     final Metadata metadata;
     final KeySupport<K> keySupport;
 
     ByteBuffer buffer;
 
-    Segment(Descriptor descriptor, SyncedOffsets syncedOffsets, Metadata metadata, KeySupport<K> keySupport)
+    Segment(Descriptor descriptor, Metadata metadata, KeySupport<K> keySupport)
     {
         this.file = descriptor.fileFor(Component.DATA);
         this.descriptor = descriptor;
-        this.syncedOffsets = syncedOffsets;
         this.metadata = metadata;
         this.keySupport = keySupport;
     }
@@ -98,7 +117,27 @@ public abstract class Segment<K, V> implements Closeable, RefCounted<Segment<K,
         }
     }
 
+    @Override
+    public int compareTo(Segment<K, V> that)
+    {
+        return this.descriptor.compareTo(that.descriptor);
+    }
+
     abstract boolean read(int offset, int size, EntrySerializer.EntryHolder<K> into);
 
-    abstract void release();
+    abstract void close(Journal<K, V> journal);
+
+    void release(Journal<K, V> journal)
+    {
+        Ref<Segment<K, V>> selfRef = selfRef();
+        Tidier tidier = (Tidier) selfRef.tidier();
+        if (journal != null)
+        {
+            // permitted to be null ONLY for tests
+            tidier.await = journal.readOrder.newBarrier();
+            tidier.await.issue();
+            tidier.executor = journal.releaser;
+        }
+        selfRef.release();
+    }
 }

src/java/org/apache/cassandra/journal/SegmentWriter.java

@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.journal;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import com.google.common.primitives.Ints;
-
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.io.util.TrackedDataOutputPlus;
-
-final class SegmentWriter<K> implements Closeable
-{
-    private final Descriptor descriptor;
-    private final KeySupport<K> keySupport;
-
-    private final InMemoryIndex<K> index;
-    private final Metadata metadata;
-
-    private final File file;
-    private FileOutputStreamPlus untrackedOut;
-    private TrackedDataOutputPlus trackedOut;
-
-    private SegmentWriter(Descriptor descriptor, KeySupport<K> keySupport)
-    {
-        this.descriptor = descriptor;
-        this.keySupport = keySupport;
-
-        index = InMemoryIndex.create(keySupport);
-        metadata = Metadata.create();
-
-        file = descriptor.fileFor(Component.DATA);
-        try
-        {
-            untrackedOut = new FileOutputStreamPlus(file);
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-        trackedOut = TrackedDataOutputPlus.wrap(untrackedOut);
-    }
-
-    static <K> SegmentWriter<K> create(Descriptor descriptor, KeySupport<K> keySupport)
-    {
-        return new SegmentWriter<>(descriptor, keySupport);
-    }
-
-    int write(K key, ByteBuffer record, Set<Integer> hosts)
-    {
-        int position = position();
-        try
-        {
-            EntrySerializer.write(key, record, hosts, keySupport, trackedOut, descriptor.userVersion);
-            index.update(key, position, position() - position);
-            metadata.update(hosts);
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-        return position;
-    }
-
-    int position()
-    {
-        return Ints.checkedCast(trackedOut.position());
-    }
-
-    @Override
-    public void close()
-    {
-        try
-        {
-            untrackedOut.flush();
-            untrackedOut.sync();
-            untrackedOut.close();
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-
-        try (SyncedOffsets syncedOffsets = SyncedOffsets.active(descriptor))
-        {
-            syncedOffsets.mark(position(), true);
-        }
-
-        index.persist(descriptor);
-        metadata.persist(descriptor);
-
-        untrackedOut = null;
-        trackedOut = null;
-    }
-}

src/java/org/apache/cassandra/journal/Segments.java

@@ -17,15 +17,13 @@
  */
 package org.apache.cassandra.journal;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.List;
 import java.util.function.Predicate;
 
 import accord.utils.Invariants;
+import accord.utils.SortedArrays.SortedArrayList;
 import org.agrona.collections.Long2ObjectHashMap;
-import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.Refs;
 
 /**
@@ -36,6 +34,7 @@ import org.apache.cassandra.utils.concurrent.Refs;
 class Segments<K, V>
 {
     private final Long2ObjectHashMap<Segment<K, V>> segments;
+    private SortedArrayList<Segment<K, V>> sorted;
 
     Segments(Long2ObjectHashMap<Segment<K, V>> segments)
     {
@@ -108,10 +107,9 @@ class Segments<K, V>
      */
     List<Segment<K, V>> allSorted(boolean asc)
     {
-        List<Segment<K, V>> segments = new ArrayList<>(this.segments.values());
-        if (asc) segments.sort(Comparator.comparing(s -> s.descriptor));
-        else segments.sort((o1, o2) -> -o1.descriptor.compareTo(o2.descriptor));
-        return segments;
+        if (sorted == null)
+            sorted = SortedArrayList.<Segment<K, V>>copyUnsorted(segments.values(), Segment[]::new);
+        return asc ? sorted : sorted.reverse();
     }
 
     void selectActive(long maxTimestamp, Collection<ActiveSegment<K, V>> into)
@@ -132,12 +130,14 @@ class Segments<K, V>
 
     ActiveSegment<K, V> oldestActive()
     {
-        Segment<K, V> oldest = null;
-        for (Segment<K, V> segment : segments.values())
-            if (segment.isActive() && (oldest == null || segment.descriptor.timestamp <= oldest.descriptor.timestamp))
-                oldest = segment;
-
-        return oldest == null ? null : oldest.asActive();
+        List<Segment<K, V>> sorted = allSorted(true);
+        for (int i = 0 ; i < sorted.size() ; ++i)
+        {
+            Segment<K, V> segment = sorted.get(i);
+            if (segment.isActive())
+                return segment.asActive();
+        }
+        return null;
     }
 
     Segment<K, V> get(long timestamp)
@@ -158,8 +158,26 @@ class Segments<K, V>
      *
      * @return a subset of segments with references to them, or {@code null} if failed to grab the refs
      */
-    @SuppressWarnings("resource")
     ReferencedSegments<K, V> selectAndReference(Predicate<Segment<K, V>> test)
+    {
+        Long2ObjectHashMap<Segment<K, V>> selectedSegments = select(test).segments;
+        Refs<Segment<K, V>> refs = null;
+        if (!selectedSegments.isEmpty())
+        {
+            refs = Refs.tryRef(selectedSegments.values());
+            if (null == refs)
+                return null;
+        }
+        return new ReferencedSegments<>(selectedSegments, refs);
+    }
+
+    /**
+     * Select segments that could potentially have an entry with the specified ids and
+     * attempt to grab references to them all.
+     *
+     * @return a subset of segments with references to them, or {@code null} if failed to grab the refs
+     */
+    Segments<K, V> select(Predicate<Segment<K, V>> test)
     {
         Long2ObjectHashMap<Segment<K, V>> selectedSegments = null;
         for (Segment<K, V> segment : segments.values())
@@ -175,14 +193,7 @@ class Segments<K, V>
         if (null == selectedSegments)
             selectedSegments = emptyMap();
 
-        Refs<Segment<K, V>> refs = null;
-        if (!selectedSegments.isEmpty())
-        {
-            refs = Refs.tryRef(selectedSegments.values());
-            if (null == refs)
-                return null;
-        }
-        return new ReferencedSegments<>(selectedSegments, refs);
+        return new Segments<>(selectedSegments);
     }
 
     static class ReferencedSegments<K, V> extends Segments<K, V> implements AutoCloseable
@@ -203,49 +214,6 @@ class Segments<K, V>
         }
     }
 
-    boolean isFlushed(RecordPointer recordPointer)
-    {
-        Segment<K, V> segment = segments.get(recordPointer.segment);
-        if (null == segment)
-            throw new IllegalArgumentException("Can not reference segment " + recordPointer.segment);
-        return segment.isFlushed(recordPointer.position);
-    }
-
-    ReferencedSegment<K, V> selectAndReference(long segmentTimestamp)
-    {
-        Segment<K, V> segment = segments.get(segmentTimestamp);
-        if (null == segment)
-            return new ReferencedSegment<>(null, null);
-        Ref<Segment<K, V>> ref = segment.tryRef();
-        if (null == ref)
-            return null;
-        return new ReferencedSegment<>(segment, ref);
-    }
-
-    static class ReferencedSegment<K, V> implements AutoCloseable
-    {
-        private final Segment<K, V> segment;
-        private final Ref<Segment<K, V>> ref;
-
-        ReferencedSegment(Segment<K, V> segment, Ref<Segment<K, V>> ref)
-        {
-            this.segment = segment;
-            this.ref = ref;
-        }
-
-        Segment<K, V> segment()
-        {
-            return segment;
-        }
-
-        @Override
-        public void close()
-        {
-            if (null != ref)
-                ref.release();
-        }
-    }
-
     private static final Long2ObjectHashMap<?> EMPTY_MAP = new Long2ObjectHashMap<>();
 
     @SuppressWarnings("unchecked")
@@ -256,6 +224,6 @@ class Segments<K, V>
 
     private static <K> Long2ObjectHashMap<K> newMap(int expectedSize)
     {
-        return new Long2ObjectHashMap<>(0, 0.65f, false);
+        return new Long2ObjectHashMap<>(expectedSize, 0.65f, false);
     }
 }

src/java/org/apache/cassandra/journal/StaticSegment.java

@@ -26,14 +26,12 @@ import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.locks.LockSupport;
 
-import accord.utils.Invariants;
 import org.agrona.collections.IntHashSet;
-import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.Closeable;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
 
 /**
@@ -45,6 +43,7 @@ import org.apache.cassandra.utils.concurrent.Ref;
 public final class StaticSegment<K, V> extends Segment<K, V>
 {
     final FileChannel channel;
+    final int fsyncLimit;
 
     private final Ref<Segment<K, V>> selfRef;
 
@@ -53,15 +52,15 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     private StaticSegment(Descriptor descriptor,
                           FileChannel channel,
                           MappedByteBuffer buffer,
-                          SyncedOffsets syncedOffsets,
                           OnDiskIndex<K> index,
                           Metadata metadata,
                           KeySupport<K> keySupport)
     {
-        super(descriptor, syncedOffsets, metadata, keySupport);
+        super(descriptor, metadata, keySupport);
         this.index = index;
 
         this.channel = channel;
+        this.fsyncLimit = metadata.fsyncLimit();
         this.buffer = buffer;
 
         selfRef = new Ref<>(this, new Tidier<>(descriptor, channel, buffer, index));
@@ -93,21 +92,17 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         if (!Component.DATA.existsFor(descriptor))
             throw new IllegalArgumentException("Data file for segment " + descriptor + " doesn't exist");
 
-        SyncedOffsets syncedOffsets = Component.SYNCED_OFFSETS.existsFor(descriptor)
-                                    ? SyncedOffsets.load(descriptor)
-                                    : SyncedOffsets.absent();
-
         Metadata metadata = Component.METADATA.existsFor(descriptor)
                           ? Metadata.load(descriptor)
-                          : Metadata.rebuildAndPersist(descriptor, keySupport, syncedOffsets.syncedOffset());
+                          : Metadata.rebuildAndPersist(descriptor, keySupport);
 
         OnDiskIndex<K> index = Component.INDEX.existsFor(descriptor)
                              ? OnDiskIndex.open(descriptor, keySupport)
-                             : OnDiskIndex.rebuildAndPersist(descriptor, keySupport, syncedOffsets.syncedOffset());
+                             : OnDiskIndex.rebuildAndPersist(descriptor, keySupport, metadata.fsyncLimit());
 
         try
         {
-            return internalOpen(descriptor, syncedOffsets, index, metadata, keySupport);
+            return internalOpen(descriptor, index, metadata, keySupport);
         }
         catch (IOException e)
         {
@@ -116,55 +111,27 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     }
 
     private static <K, V> StaticSegment<K, V> internalOpen(
-        Descriptor descriptor, SyncedOffsets syncedOffsets, OnDiskIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
+        Descriptor descriptor, OnDiskIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
     throws IOException
     {
         File file = descriptor.fileFor(Component.DATA);
         FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
         MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
-        return new StaticSegment<>(descriptor, channel, buffer, syncedOffsets, index, metadata, keySupport);
+        return new StaticSegment<>(descriptor, channel, buffer, index, metadata, keySupport);
     }
 
-    @Override
-    public void close()
+    public void close(Journal<K, V> journal)
     {
-        try
-        {
-            channel.close();
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException("Could not close static segment " + descriptor, e);
-        }
-
-        release();
+        release(journal);
     }
 
     /**
      * Waits until this segment is unreferenced, closes it, and deltes all files associated with it.
      */
-    void discard()
+    void discard(Journal<K, V> journal)
     {
-        // TODO: consider moving deletion logic to Tidier instead of busy-looping here
-        waitUntilUnreferenced();
-        close();
-        for (Component component : Component.values())
-        {
-            File file = descriptor.fileFor(component);
-            if (file.exists())
-                file.delete();
-        }
-    }
-
-    public void waitUntilUnreferenced()
-    {
-        while (true)
-        {
-            if (selfRef.globalCount() == 1)
-                return;
-
-            LockSupport.parkNanos(100);
-        }
+        ((Tidier)selfRef.tidier()).discard = true;
+        close(journal);
     }
 
     @Override
@@ -179,24 +146,25 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         return selfRef.ref();
     }
 
-    @Override
-    void release()
-    {
-        selfRef.release();
-    }
-
     @Override
     public String toString()
     {
         return "StaticSegment{" + descriptor + '}';
     }
 
-    private static final class Tidier<K> implements Tidy
+    @Override
+    public Ref<Segment<K, V>> selfRef()
+    {
+        return selfRef;
+    }
+
+    private static final class Tidier<K> extends Segment.Tidier implements Tidy
     {
         private final Descriptor descriptor;
         private final FileChannel channel;
         private final ByteBuffer buffer;
         private final Index<K> index;
+        boolean discard;
 
         Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer, Index<K> index)
         {
@@ -207,11 +175,21 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         }
 
         @Override
-        public void tidy()
+        void onUnreferenced()
         {
             FileUtils.clean(buffer);
             FileUtils.closeQuietly(channel);
             index.close();
+            if (discard)
+            {
+                Throwable fail = null;
+                for (Component component : Component.VALUES)
+                {
+                    try { descriptor.fileFor(component).deleteIfExists(); }
+                    catch (Throwable t) { fail = Throwables.merge(fail, t); }
+                }
+                Throwables.maybeFail(fail);
+            }
         }
 
         @Override
@@ -259,13 +237,9 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     boolean read(int offset, int size, EntrySerializer.EntryHolder<K> into)
     {
         ByteBuffer duplicate = buffer.duplicate().position(offset).limit(offset + size);
-        try (DataInputBuffer in = new DataInputBuffer(duplicate, false))
+        try
         {
-            if (!EntrySerializer.tryRead(into, keySupport, duplicate, in, syncedOffsets.syncedOffset(), descriptor.userVersion))
-                return false;
-
-            Invariants.checkState(in.available() == 0);
-            return true;
+            return 0 <= EntrySerializer.tryRead(into, keySupport, duplicate, fsyncLimit, descriptor.userVersion);
         }
         catch (IOException e)
         {
@@ -278,7 +252,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
      */
     void forEachRecord(RecordConsumer<K> consumer)
     {
-        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, syncedOffsets.syncedOffset()))
+        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, fsyncLimit))
         {
             while (reader.advance())
             {
@@ -389,13 +363,11 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     static final class SequentialReader<K> extends Reader<K>
     {
         private final int fsyncedLimit; // exclusive
-        private final DataInputBuffer in;
 
         SequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
         {
             super(descriptor, keySupport);
             this.fsyncedLimit = fsyncedLimit;
-            in = new DataInputBuffer(buffer, false);
         }
 
         @Override
@@ -413,8 +385,10 @@ public final class StaticSegment<K, V> extends Segment<K, V>
             offset = buffer.position();
             try
             {
-                if (!EntrySerializer.tryRead(holder, keySupport, buffer, in, fsyncedLimit, descriptor.userVersion))
+                int length = EntrySerializer.tryRead(holder, keySupport, buffer.duplicate(), fsyncedLimit, descriptor.userVersion);
+                if (length < 0)
                     return eof();
+                buffer.position(offset + length);
             }
             catch (IOException e)
             {

src/java/org/apache/cassandra/journal/SyncedOffsets.java

@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.journal;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.NoSuchFileException;
-import java.util.zip.CRC32;
-
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.Closeable;
-import org.apache.cassandra.utils.Crc;
-
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
-
-/**
- * Keeps track of fsynced limits of a data file. Enables us to treat invalid
- * records that are known to have been fsynced to disk differently from those
- * that aren't.
- * <p/>
- * On disk representation is a sequence of 2-int tuples of {synced offset, CRC32(synced offset)}
- */
-interface SyncedOffsets extends Closeable
-{
-    /**
-     * @return furthest known synced offset
-     */
-    int syncedOffset();
-
-    /**
-     * Record an offset as synced to disk.
-     *
-     * @param offset the offset into datafile, up to which contents have been fsynced (exclusive)
-     */
-    void mark(int offset, boolean fsync);
-
-    void fsync();
-
-    @Override
-    default void close()
-    {
-    }
-
-    /**
-     * @return a disk-backed synced offset tracker for a new {@link ActiveSegment}
-     */
-    static Active active(Descriptor descriptor)
-    {
-        return new Active(descriptor);
-    }
-
-    /**
-     * Load an existing log of synced offsets from disk into an immutable instance.
-     */
-    static Static load(Descriptor descriptor)
-    {
-        return Static.load(descriptor);
-    }
-
-    /**
-     * @return a placeholder instance in case this component is missing
-     */
-    static Absent absent()
-    {
-        return Absent.INSTANCE;
-    }
-
-    /**
-     * Single-threaded, file-based list of synced offsets.
-     */
-    final class Active implements SyncedOffsets
-    {
-        private final Descriptor descriptor;
-
-        private final FileOutputStreamPlus output;
-        private volatile int syncedOffset;
-
-        private Active(Descriptor descriptor)
-        {
-            this.descriptor = descriptor;
-
-            File file = descriptor.fileFor(Component.SYNCED_OFFSETS);
-            if (file.exists())
-                throw new IllegalArgumentException("Synced offsets file " + file + " already exists");
-
-            try
-            {
-                output = file.newOutputStream(File.WriteMode.OVERWRITE);
-            }
-            catch (UncheckedIOException | FSWriteError e)
-            {
-                // extract original cause and throw as JournalWriteError
-                throw new JournalWriteError(descriptor, file, e.getCause());
-            }
-            catch (NoSuchFileException e)
-            {
-                throw new AssertionError(); // unreachable
-            }
-        }
-
-        @Override
-        public int syncedOffset()
-        {
-            return syncedOffset;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            if (offset < syncedOffset)
-                throw new IllegalArgumentException("offset " + offset + " is smaller than previous mark " + offset);
-
-            CRC32 crc = Crc.crc32();
-            updateChecksumInt(crc, offset);
-
-            try
-            {
-                output.writeInt(offset);
-                output.writeInt((int) crc.getValue());
-                output.flush();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-
-            syncedOffset = offset;
-            if (fsync) fsync();
-        }
-
-        public void fsync()
-        {
-            try
-            {
-                output.sync();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-        }
-
-        @Override
-        public void close()
-        {
-            fsync();
-
-            try
-            {
-                output.close();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-        }
-    }
-
-    final class Static implements SyncedOffsets
-    {
-        private final int syncedOffset;
-
-        static Static load(Descriptor descriptor)
-        {
-            File file = descriptor.fileFor(Component.SYNCED_OFFSETS);
-            if (!file.exists())
-                throw new IllegalArgumentException("Synced offsets file " + file + " doesn't exist");
-
-            int syncedOffset = 0;
-            try (RandomAccessReader reader = RandomAccessReader.open(file))
-            {
-                CRC32 crc = Crc.crc32();
-                while (reader.bytesRemaining() >= 8)
-                {
-                    int offset = reader.readInt();
-                    updateChecksumInt(crc, offset);
-                    int readCrc = reader.readInt();
-                    if (readCrc != (int) crc.getValue())
-                        break;
-                    syncedOffset = offset;
-                    Crc.initialize(crc);
-                }
-            }
-            catch (Throwable t)
-            {
-                throw new JournalReadError(descriptor, file, t);
-            }
-
-            return new Static(syncedOffset);
-        }
-
-        Static(int offset)
-        {
-            this.syncedOffset = offset;
-        }
-
-        @Override
-        public int syncedOffset()
-        {
-            return syncedOffset;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void fsync()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    final class Absent implements SyncedOffsets
-    {
-        static final Absent INSTANCE = new Absent();
-
-        @Override
-        public int syncedOffset()
-        {
-            return 0;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void fsync()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-}

src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java

@@ -26,7 +26,7 @@ import com.codahale.metrics.Histogram;
 import static org.apache.cassandra.metrics.CacheMetrics.TYPE_NAME;
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
-public class AccordStateCacheMetrics extends CacheAccessMetrics
+public class AccordCacheMetrics extends CacheAccessMetrics
 {
     public static final String OBJECT_SIZE = "ObjectSize";
 
@@ -36,7 +36,7 @@ public class AccordStateCacheMetrics extends CacheAccessMetrics
 
     private final String scope;
 
-    public AccordStateCacheMetrics(String scope)
+    public AccordCacheMetrics(String scope)
     {
         super(new DefaultNameFactory(TYPE_NAME, scope));
         objectSize = Metrics.histogram(factory.createMetricName(OBJECT_SIZE), false);

src/java/org/apache/cassandra/metrics/AccordMetrics.java

@@ -49,6 +49,7 @@ public class AccordMetrics
     public static final String PROGRESS_LOG_SIZE = "ProgressLogSize";
 
     public static final String DEPENDENCIES = "Dependencies";
+    public static final String EPHEMERAL = "Ephemeral";
     public static final String FAST_PATHS = "FastPaths";
     public static final String SLOW_PATHS = "SlowPaths";
     public static final String PREEMPTS = "Preempts";

src/java/org/apache/cassandra/schema/TableId.java

@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nullable;
 
+import org.agrona.collections.Hashing;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.commons.lang3.ArrayUtils;
@@ -41,6 +42,7 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.UUIDGen;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -51,18 +53,24 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
  * This is essentially a UUID, but we wrap it as it's used quite a bit in the code and having a nicely named class make
  * the code more readable.
  */
-public class TableId implements Comparable<TableId>
+public final class TableId implements Comparable<TableId>
 {
     public static final long MAGIC = 1956074401491665062L;
     public static final long EMPTY_SIZE = ObjectSizes.measureDeep(new UUID(0, 0));
 
     private static final ConcurrentHashMap<TableId, TableId> internCache = new ConcurrentHashMap<>();
 
-    private final UUID id;
+    final long msb, lsb;
 
     private TableId(UUID id)
     {
-        this.id = id;
+        this(id.getMostSignificantBits(), id.getLeastSignificantBits());
+    }
+
+    private TableId(long msb, long lsb)
+    {
+        this.msb = msb;
+        this.lsb = lsb;
     }
 
     public static TableId fromUUID(UUID id)
@@ -70,6 +78,11 @@ public class TableId implements Comparable<TableId>
         return new TableId(id);
     }
 
+    public static TableId fromRaw(long msb, long lsb)
+    {
+        return new TableId(msb, lsb);
+    }
+
     // TODO: should we be using UUID.randomUUID()?
     public static TableId generate()
     {
@@ -145,43 +158,46 @@ public class TableId implements Comparable<TableId>
 
     public String toHexString()
     {
-        return ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(id));
+        return ByteBufferUtil.bytesToHex(ByteBuffer.wrap(UUIDGen.decompose(msb, lsb)));
     }
 
     public UUID asUUID()
     {
-        return id;
+        return new UUID(msb, lsb);
     }
 
     @Override
-    public final int hashCode()
+    public int hashCode()
     {
-        return id.hashCode();
+        return Hashing.hash(msb ^ lsb);
     }
 
     @Override
     public final boolean equals(Object o)
     {
-        return this == o || (o instanceof TableId && this.id.equals(((TableId) o).id));
+        if (o == this | o == null) return o == this;
+        if (o.getClass() != TableId.class) return false;
+        TableId that = (TableId) o;
+        return this.msb == that.msb && this.lsb == that.lsb;
     }
 
     @Override
     public String toString()
     {
-        return id.toString();
+        return new UUID(msb, lsb).toString();
     }
 
     public void serialize(DataOutput out) throws IOException
     {
-        out.writeLong(id.getMostSignificantBits());
-        out.writeLong(id.getLeastSignificantBits());
+        out.writeLong(msb);
+        out.writeLong(lsb);
     }
 
     public <V> int serialize(V dst, ValueAccessor<V> accessor, int offset)
     {
         int position = offset;
-        position += accessor.putLong(dst, position, id.getMostSignificantBits());
-        position += accessor.putLong(dst, position, id.getLeastSignificantBits());
+        position += accessor.putLong(dst, position, msb);
+        position += accessor.putLong(dst, position, lsb);
         return position - offset;
     }
 
@@ -197,12 +213,12 @@ public class TableId implements Comparable<TableId>
 
     public static TableId deserialize(DataInput in) throws IOException
     {
-        return new TableId(new UUID(in.readLong(), in.readLong()));
+        return new TableId(in.readLong(), in.readLong());
     }
 
     public static <V> TableId deserialize(V src, ValueAccessor<V> accessor, int offset)
     {
-        return new TableId(new UUID(accessor.getLong(src, offset), accessor.getLong(src, offset + TypeSizes.LONG_SIZE)));
+        return new TableId(accessor.getLong(src, offset), accessor.getLong(src, offset + TypeSizes.LONG_SIZE));
     }
 
     public TableId intern()
@@ -212,9 +228,10 @@ public class TableId implements Comparable<TableId>
     }
 
     @Override
-    public int compareTo(TableId o)
+    public int compareTo(TableId that)
     {
-        return id.compareTo(o.id);
+        int c = Long.compare(this.msb, that.msb);
+        return c != 0 ? c : Long.compare(this.lsb, that.lsb);
     }
 
     public static final IVersionedSerializer<TableId> serializer = new IVersionedSerializer<>()
@@ -263,5 +280,4 @@ public class TableId implements Comparable<TableId>
     {
         ScheduledExecutors.scheduledFastTasks.scheduleSelfRecurring(internCache::clear, 1, TimeUnit.HOURS);
     }
-
 }

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -179,6 +179,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.concat;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordEphemeralReadEnabledEnabled;
 import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
@@ -205,6 +206,7 @@ import static org.apache.cassandra.service.consensus.migration.ConsensusMigratio
 import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.splitMutationsIntoAccordAndNormal;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
+import static org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode.none;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
 import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
@@ -2199,7 +2201,7 @@ public class StorageProxy implements StorageProxyMBean
         consistencyLevel = consistencyLevelForAccordRead(cm, group, consistencyLevel);
         TxnRead read = TxnRead.createSerialRead(group.queries, consistencyLevel);
         Txn.Kind kind = Read;
-        if (transactionalMode == TransactionalMode.full && DatabaseDescriptor.getAccordEphemeralReadEnabledEnabled() && group.queries.size() == 1)
+        if (transactionalMode == TransactionalMode.full && getAccordEphemeralReadEnabledEnabled() && group.queries.size() == 1 && group.metadata().params.transactionalMigrationFrom == none)
             kind = EphemeralRead;
         Txn txn = new Txn.InMemory(kind, read.keys(), read, TxnQuery.ALL, null);
         AsyncTxnResult asyncTxnResult = AccordService.instance().coordinateAsync(txn, consistencyLevel, requestTime);

src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java

@@ -0,0 +1,664 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import accord.utils.ArrayBuffers.BufferList;
+import accord.utils.IntrusiveLinkedListNode;
+import accord.utils.Invariants;
+import accord.utils.async.Cancellable;
+import org.apache.cassandra.service.accord.AccordCache.Adapter;
+import org.apache.cassandra.utils.ObjectSizes;
+
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADING;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.SAVING;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_LOAD;
+
+/**
+ * Global (per CommandStore) state of a cached entity (Command or CommandsForKey).
+ */
+public class AccordCacheEntry<K, V> extends IntrusiveLinkedListNode
+{
+    public enum Status
+    {
+        UNINITIALIZED,
+        WAITING_TO_LOAD(UNINITIALIZED),
+        LOADING(WAITING_TO_LOAD),
+        /**
+         * Consumers should never see this state
+         */
+        FAILED_TO_LOAD(LOADING),
+
+        LOADED(true, false, UNINITIALIZED, LOADING),
+        MODIFIED(true, false, LOADED),
+        SAVING(true, true, MODIFIED),
+
+        /**
+         * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization,
+         * or commit log has been stopped.
+         */
+        FAILED_TO_SAVE(true, true, SAVING),
+
+        UNUSED, // spacing to permit easier bit masks
+
+        EVICTED(WAITING_TO_LOAD, LOADING, LOADED, FAILED_TO_LOAD),
+        ;
+
+        static final Status[] VALUES = values();
+        static
+        {
+            MODIFIED.permittedFrom |= 1 << MODIFIED.ordinal();
+            MODIFIED.permittedFrom |= 1 << SAVING.ordinal();
+            MODIFIED.permittedFrom |= 1 << FAILED_TO_SAVE.ordinal();
+            LOADED.permittedFrom |= 1 << SAVING.ordinal();
+            LOADED.permittedFrom |= 1 << MODIFIED.ordinal();
+            for (Status status : VALUES)
+            {
+                Invariants.checkState((status.ordinal() & IS_LOADED) != 0 == status.loaded);
+                Invariants.checkState(((status.ordinal() & IS_LOADED) != 0 && (status.ordinal() & IS_NESTED) != 0) == status.nested);
+            }
+        }
+
+        final boolean loaded;
+        final boolean nested;
+        int permittedFrom;
+
+        Status(Status ... statuses)
+        {
+            this(false, false, statuses);
+        }
+
+        Status(boolean loaded, boolean nested, Status ... statuses)
+        {
+            this.loaded = loaded;
+            this.nested = nested;
+            for (Status status : statuses)
+                permittedFrom |= 1 << status.ordinal();
+        }
+    }
+
+    static final int STATUS_MASK = 0x0000001F;
+    static final int SHRUNK = 0x00000040;
+    static final int NO_EVICT = 0x00000020;
+    static final int IS_LOADED = 0x4;
+    static final int IS_NESTED = 0x2; // only valid to test if already tested NORMAL
+    static final int IS_LOADING_OR_WAITING_MASK = 0x6; // only valid to test if already tested NORMAL
+    static final int IS_LOADING_OR_WAITING = 0x2; // only valid to test if already tested NORMAL
+    static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null));
+
+    private final K key;
+    final AccordCache.Type<K, V, ?>.Instance owner;
+
+    private Object state;
+    private int status;
+    int sizeOnHeap;
+    private volatile int references;
+    private static final AtomicIntegerFieldUpdater<AccordCacheEntry> referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references");
+
+    AccordCacheEntry(K key, AccordCache.Type<K, V, ?>.Instance owner)
+    {
+        this.key = key;
+        this.owner = owner;
+    }
+
+    void unlink()
+    {
+        remove();
+    }
+
+    boolean isUnqueued()
+    {
+        return isFree();
+    }
+
+    public K key()
+    {
+        return key;
+    }
+
+    public int references()
+    {
+        return references;
+    }
+
+    public int increment()
+    {
+        return referencesUpdater.incrementAndGet(this);
+    }
+
+    public int decrement()
+    {
+        return referencesUpdater.decrementAndGet(this);
+    }
+
+    boolean isLoaded()
+    {
+        return (status & IS_LOADED) != 0;
+    }
+
+    boolean isNested()
+    {
+        Invariants.checkState(isLoaded());
+        return (status & IS_NESTED) != 0;
+    }
+
+    boolean isShrunk()
+    {
+        return (status & SHRUNK) != 0;
+    }
+
+    public boolean is(Status status)
+    {
+        return (this.status & STATUS_MASK) == status.ordinal();
+    }
+
+    boolean isLoadingOrWaiting()
+    {
+        return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING;
+    }
+
+    public boolean isComplete()
+    {
+        return !is(LOADING) && !is(SAVING);
+    }
+
+    int noEvictGeneration()
+    {
+        Invariants.checkState(isNoEvict());
+        return (status >>> 8) & 0xffff;
+    }
+
+    int noEvictMaxAge()
+    {
+        Invariants.checkState(isNoEvict());
+        return status >>> 24;
+    }
+
+    boolean isNoEvict()
+    {
+        return (status & NO_EVICT) != 0;
+    }
+
+    int sizeOnHeap()
+    {
+        return sizeOnHeap;
+    }
+
+    void updateSize(AccordCache.Type<K, V, ?> parent)
+    {
+        // TODO (expected): we aren't weighing the keys
+        int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter()));
+        parent.updateSize(newSizeOnHeap, newSizeOnHeap - sizeOnHeap, references == 0, true);
+        sizeOnHeap = newSizeOnHeap;
+    }
+
+    void initSize(AccordCache.Type<K, V, ?> parent)
+    {
+        // TODO (expected): we aren't weighing the keys
+        sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE);
+        parent.updateSize(sizeOnHeap, sizeOnHeap, false, false);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Node{" + status() +
+               ", key=" + key() +
+               ", references=" + references +
+               "}@" + Integer.toHexString(System.identityHashCode(this));
+    }
+
+    public Status status()
+    {
+        return Status.VALUES[(status & STATUS_MASK)];
+    }
+
+    private void setStatus(Status newStatus)
+    {
+        Invariants.checkState((newStatus.permittedFrom & (1 << (status & STATUS_MASK))) != 0, "%s not permitted from %s", newStatus, status());
+        status &= ~STATUS_MASK;
+        status |= newStatus.ordinal();
+        Invariants.checkState(status() == newStatus);
+    }
+
+    public void initialize(V value)
+    {
+        Invariants.checkState(state == null);
+        setStatus(LOADED);
+        state = value;
+    }
+
+    public void readyToLoad()
+    {
+        Invariants.checkState(state == null);
+        setStatus(WAITING_TO_LOAD);
+        state = new WaitingToLoad();
+    }
+
+    public void markNoEvict(int generation, int maxAge)
+    {
+        Invariants.checkState((maxAge & ~0xff) == 0);
+        Invariants.checkState((generation & ~0xffff) == 0);
+        status |= NO_EVICT;
+        status |= generation << 8;
+        status |= maxAge << 24;
+    }
+
+    public LoadingOrWaiting loadingOrWaiting()
+    {
+        return (LoadingOrWaiting)state;
+    }
+
+    void notifyListeners(BiConsumer<AccordCache.Listener<K, V>, AccordCacheEntry<K, V>> notify)
+    {
+        owner.notifyListeners(notify, this);
+    }
+
+    public interface OnLoaded
+    {
+        <K, V> void onLoaded(AccordCacheEntry<K, V> state, V value, Throwable fail);
+
+        static OnLoaded immediate()
+        {
+            return new OnLoaded()
+            {
+                @Override
+                public <K, V> void onLoaded(AccordCacheEntry<K, V> state, V value, Throwable fail)
+                {
+                    if (fail == null) state.loaded(value);
+                    else state.failedToLoad();
+                }
+            };
+        }
+    }
+
+    public interface OnSaved
+    {
+        <K, V> void onSaved(AccordCacheEntry<K, V> state, Object identity, Throwable fail);
+
+        static OnSaved immediate()
+        {
+            return new OnSaved()
+            {
+                @Override
+                public <K, V> void onSaved(AccordCacheEntry<K, V> state, Object identity, Throwable fail)
+                {
+                    state.saved(identity, fail);
+                }
+            };
+        }
+    }
+
+    public <P> Loading load(BiFunction<P, Runnable, Cancellable> loadExecutor, P param, Adapter<K, V, ?> adapter, OnLoaded onLoaded)
+    {
+        Invariants.checkState(is(WAITING_TO_LOAD), "%s", this);
+        Loading loading = ((WaitingToLoad)state).load(loadExecutor.apply(param, () -> {
+            V result;
+            try
+            {
+                result = adapter.load(owner.commandStore, key);
+            }
+            catch (Throwable t)
+            {
+                onLoaded.onLoaded(this, null, t);
+                throw t;
+            }
+            onLoaded.onLoaded(this, result, null);
+        }));
+        setStatus(LOADING);
+        state = loading;
+        return loading;
+    }
+
+    public Loading testLoad()
+    {
+        Invariants.checkState(is(WAITING_TO_LOAD));
+        Loading loading = ((WaitingToLoad)state).load(() -> {});
+        setStatus(LOADING);
+        state = loading;
+        return loading;
+    }
+
+    public Loading loading()
+    {
+        Invariants.checkState(is(LOADING), "%s", this);
+        return (Loading) state;
+    }
+
+    // must own the cache's lock when invoked. this is true of most methods in the class,
+    // but this one is less obvious so named as to draw attention
+    public V getExclusive()
+    {
+        Invariants.checkState(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread());
+        Invariants.checkState(isLoaded(), "%s", this);
+        if (isShrunk())
+        {
+            AccordCache.Type<K, V, ?> parent = owner.parent();
+            inflate(key, parent.adapter());
+            updateSize(parent);
+        }
+
+        return (V)unwrap();
+    }
+
+    private Object unwrap()
+    {
+        return isNested() ? ((Nested)state).state : state;
+    }
+
+    // must own the cache's lock when invoked
+    void setExclusive(V value)
+    {
+        if (value == state)
+            return;
+
+        Saving cancel = is(SAVING) ? ((Saving)state) : null;
+        setStatus(MODIFIED);
+        state = value;
+        updateSize(owner.parent());
+        // TODO (expected): do we want to always cancel in-progress saving?
+        if (cancel != null)
+            cancel.saving.cancel();
+    }
+
+    public void loaded(V value)
+    {
+        setStatus(LOADED);
+        state = value;
+        updateSize(owner.parent());
+    }
+
+    public void testLoaded(V value)
+    {
+        setStatus(LOADED);
+        state = value;
+    }
+
+    public void failedToLoad()
+    {
+        setStatus(FAILED_TO_LOAD);
+        state = null;
+    }
+
+    boolean tryShrink()
+    {
+        if (!isLoaded())
+            return false;
+
+        AccordCache.Type<K, V, ?> parent = owner.parent();
+        if (!tryShrink(key, parent.adapter()))
+            return false;
+        updateSize(parent);
+        return true;
+    }
+
+    V tryGetFull()
+    {
+        return isShrunk() ? null : (V)unwrap();
+    }
+
+    Object tryGetShrunk()
+    {
+        return isShrunk() ? unwrap() : null;
+    }
+
+    boolean isNull()
+    {
+        return state == null;
+    }
+
+    /**
+     * Submits a save runnable to the specified executor. When the runnable
+     * has completed, the state save will have either completed or failed.
+     */
+    @VisibleForTesting
+    void save(Function<Runnable, Cancellable> saveExecutor, Adapter<K, V, ?> adapter, OnSaved onSaved)
+    {
+        V full = isShrunk() ? null : (V)state;
+        Object shrunk = isShrunk() ? state : null;
+        Runnable save = adapter.save(owner.commandStore, key, full, shrunk);
+        if (null == save) // null mutation -> null Runnable -> no change on disk
+        {
+            setStatus(LOADED);
+        }
+        else
+        {
+            setStatus(SAVING);
+            Object identity = new Object();
+            Cancellable saving = saveExecutor.apply(() -> {
+                try
+                {
+                    save.run();
+                }
+                catch (Throwable t)
+                {
+                    onSaved.onSaved(this, identity, t);
+                    throw t;
+                }
+                onSaved.onSaved(this, identity, null);
+            });
+            state = new Saving(saving, identity, state);
+        }
+    }
+
+    boolean saved(Object identity, Throwable fail)
+    {
+        if (!is(SAVING))
+            return false;
+
+        Saving saving = (Saving) state;
+        if (saving.identity != identity)
+            return false;
+
+        if (fail != null)
+        {
+            setStatus(FAILED_TO_SAVE);
+            state = new FailedToSave(fail, ((Saving)state).state);
+            return false;
+        }
+        else
+        {
+            setStatus(LOADED);
+            state = saving.state;
+            return true;
+        }
+    }
+
+    protected void saved()
+    {
+        Invariants.checkState(is(MODIFIED));
+        setStatus(LOADED);
+    }
+
+    public Cancellable saving()
+    {
+        return ((Saving)state).saving;
+    }
+
+    public AccordCacheEntry<K, V> evicted()
+    {
+        setStatus(EVICTED);
+        state = null;
+        return this;
+    }
+
+    public Throwable failure()
+    {
+        return ((FailedToSave)state).cause;
+    }
+
+    private boolean tryShrink(K key, Adapter<K, V, ?> adapter)
+    {
+        Invariants.checkState(!isNested());
+        if (isShrunk() || state == null)
+            return false;
+
+        Object update = adapter.fullShrink(key, (V)state);
+        if (update == null || update == state)
+            return false;
+
+        state = update;
+        status |= SHRUNK;
+        return true;
+    }
+
+    private void inflate(K key, Adapter<K, V, ?> adapter)
+    {
+        Invariants.checkState(isShrunk());
+        if (isNested())
+        {
+            Nested nested = (Nested) state;
+            nested.state = adapter.inflate(key, nested.state);
+        }
+        else
+        {
+            state = adapter.inflate(key, state);
+        }
+        status &= ~SHRUNK;
+    }
+
+    private long estimateOnHeapSize(Adapter<K, V, ?> adapter)
+    {
+        Object current = unwrap();
+        if (current == null) return 0;
+        else if (isShrunk()) return adapter.estimateShrunkHeapSize(current);
+        return adapter.estimateHeapSize((V)current);
+    }
+
+    public static abstract class LoadingOrWaiting
+    {
+        Collection<AccordTask<?>> waiters;
+
+        public LoadingOrWaiting()
+        {
+        }
+
+        public LoadingOrWaiting(Collection<AccordTask<?>> waiters)
+        {
+            this.waiters = waiters;
+        }
+
+        public Collection<AccordTask<?>> waiters()
+        {
+            return waiters != null ? waiters : Collections.emptyList();
+        }
+
+        public BufferList<AccordTask<?>> copyWaiters()
+        {
+            BufferList<AccordTask<?>> list = new BufferList<>();
+            if (waiters != null)
+                list.addAll(waiters);
+            return list;
+        }
+
+        public void add(AccordTask<?> waiter)
+        {
+            if (waiters == null)
+                waiters = new ArrayList<>();
+            waiters.add(waiter);
+        }
+
+        public void remove(AccordTask<?> waiter)
+        {
+            if (waiters != null)
+            {
+                waiters.remove(waiter);
+                if (waiters.isEmpty())
+                    waiters = null;
+            }
+        }
+    }
+
+    static class WaitingToLoad extends LoadingOrWaiting
+    {
+        public Loading load(Cancellable loading)
+        {
+            Invariants.paranoid(waiters == null || !waiters.isEmpty());
+            Loading result = new Loading(waiters, loading);
+            waiters = Collections.emptyList();
+            return result;
+        }
+    }
+
+    static class Loading extends LoadingOrWaiting
+    {
+        public final Cancellable loading;
+
+        public Loading(Collection<AccordTask<?>> waiters, Cancellable loading)
+        {
+            super(waiters);
+            this.loading = loading;
+        }
+    }
+
+    static class Nested
+    {
+        Object state;
+    }
+
+    static class Saving extends Nested
+    {
+        final Cancellable saving;
+        final Object identity;
+
+        Saving(Cancellable saving, Object identity, Object state)
+        {
+            this.saving = saving;
+            this.identity = identity;
+            this.state = state;
+        }
+    }
+
+    static class FailedToSave extends Nested
+    {
+        final Throwable cause;
+
+        FailedToSave(Throwable cause, Object state)
+        {
+            this.cause = cause;
+            this.state = state;
+        }
+
+        public Throwable failure()
+        {
+            return cause;
+        }
+    }
+
+    public static <K, V> AccordCacheEntry<K, V> createReadyToLoad(K key, AccordCache.Type<K, V, ?>.Instance owner)
+    {
+        AccordCacheEntry<K, V> node = new AccordCacheEntry<>(key, owner);
+        node.readyToLoad();
+        return node;
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordCachingState.java

@@ -1,651 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.util.concurrent.Callable;
-import java.util.function.Function;
-import java.util.function.ToLongFunction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.primitives.Ints;
-
-import accord.utils.IntrusiveLinkedListNode;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncResults.RunnableResult;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.utils.ObjectSizes;
-
-import static java.lang.String.format;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_SAVE;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.MODIFIED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.SAVING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.UNINITIALIZED;
-
-/**
- * Global (per CommandStore) state of a cached entity (Command or CommandsForKey).
- */
-public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
-{
-    static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCachingState<>(null, 0, null));
-
-    public interface Factory<K, V>
-    {
-        AccordCachingState<K, V> create(K key, int index);
-    }
-
-    static <K, V> Factory<K, V> defaultFactory()
-    {
-        return AccordCachingState::new;
-    }
-
-    private final K key;
-    private State<K, V> state;
-
-    int references = 0;
-    int lastQueriedEstimatedSizeOnHeap = 0;
-    final int index;
-    private boolean shouldUpdateSize;
-
-    AccordCachingState(K key, int index)
-    {
-        this.key = key;
-        Invariants.checkArgument(index >= 0);
-        this.index = index;
-        //noinspection unchecked
-        this.state = (State<K, V>) Uninitialized.instance;
-    }
-
-    private AccordCachingState(K key, int index, State<K, V> state)
-    {
-        this.key = key;
-        this.index = index;
-        this.state = state;
-    }
-
-    void unlink()
-    {
-        remove();
-    }
-
-    boolean isLinked()
-    {
-        return !isFree();
-    }
-
-    public K key()
-    {
-        return key;
-    }
-
-    public int referenceCount()
-    {
-        return references;
-    }
-
-    boolean isLoaded()
-    {
-        return status().isLoaded();
-    }
-
-    public boolean isComplete()
-    {
-        return status().isComplete();
-    }
-
-    int lastQueriedEstimatedSizeOnHeap()
-    {
-        return lastQueriedEstimatedSizeOnHeap;
-    }
-
-    int estimatedSizeOnHeap(ToLongFunction<V> estimator)
-    {
-        shouldUpdateSize = false;   // TODO (expected): probably not the safest place to clear need to compute size
-        return lastQueriedEstimatedSizeOnHeap = Ints.checkedCast(EMPTY_SIZE + estimateStateOnHeapSize(estimator));
-    }
-
-    long estimatedSizeOnHeapDelta(ToLongFunction<V> estimator)
-    {
-        long prevSize = lastQueriedEstimatedSizeOnHeap;
-        return estimatedSizeOnHeap(estimator) - prevSize;
-    }
-
-    boolean shouldUpdateSize()
-    {
-        return shouldUpdateSize;
-    }
-
-    @Override
-    public String toString()
-    {
-        return "Node{" + state.status() +
-               ", key=" + key() +
-               ", references=" + references +
-               "}@" + Integer.toHexString(System.identityHashCode(this));
-    }
-
-    public Status status()
-    {
-        return complete().status();
-    }
-
-    State<K, V> complete()
-    {
-        return state.isCompleteable() ? state(state.complete()) : state;
-    }
-
-    /**
-     * Submits a load runnable to the specified executor. When the runnable
-     * has completed, the state load will have either completed or failed.
-     */
-    public AsyncChain<V> load(ExecutorPlus executor, Function<K, V> loadFunction)
-    {
-        Loading<K, V> loading = state.load(key, loadFunction);
-        executor.submit(loading);
-        state(loading);
-        return loading;
-    }
-
-    public void initialize(V value)
-    {
-        state(state.initialize(value));
-    }
-
-    protected State<K, V> state(State<K, V> next)
-    {
-        State<K, V> prev = state;
-        if (prev != next)   // TODO (expected): we change state to transition the cache state machine but often keep payload the same - so shouldn't recompute
-            shouldUpdateSize = true;
-        return state = next;
-    }
-
-    @VisibleForTesting
-    protected State<K, V> state()
-    {
-        return state;
-    }
-
-    public AsyncChain<V> loading()
-    {
-        // do *not* attempt to complete, to prevent races where the caller found a pending load, attempts
-        // to register a callback, but gets an exception because the load completed in the meantime
-        return state.loading();
-    }
-
-    public V get()
-    {
-        return complete().get();
-    }
-
-    public void set(V value)
-    {
-        shouldUpdateSize = true;
-        state(complete().set(value));
-    }
-
-    /**
-     * Submits a save runnable to the specified executor. When the runnable
-     * has completed, the state save will have either completed or failed.
-     */
-    @VisibleForTesting
-    public void save(ExecutorPlus executor, Function<?, Runnable> saveFunction)
-    {
-        @SuppressWarnings("unchecked")
-        State<K, V> savingOrLoaded = state.save((Function<V, Runnable>) saveFunction);
-        if (savingOrLoaded.status() == SAVING)
-            executor.submit(savingOrLoaded.saving());
-        state(savingOrLoaded);
-    }
-
-    public AsyncChain<Void> saving()
-    {
-        // do *not* attempt to complete, to prevent races where the caller found a pending save, attempts
-        // to register a callback, but gets an exception because the save completed in the meantime
-        return state.saving();
-    }
-
-    public AccordCachingState<K, V> reset()
-    {
-        state(state.reset());
-        return this;
-    }
-
-    public Throwable failure()
-    {
-        return complete().failure();
-    }
-
-    public void markEvicted()
-    {
-        state(complete().evict());
-        lastQueriedEstimatedSizeOnHeap = 0;
-        shouldUpdateSize = false;
-    }
-
-    long estimateStateOnHeapSize(ToLongFunction<V> estimateFunction)
-    {
-        return state.estimateOnHeapSize(estimateFunction);
-    }
-
-    public enum Status
-    {
-        UNINITIALIZED,
-        LOADING,
-        LOADED,
-        FAILED_TO_LOAD,
-        MODIFIED,
-        SAVING,
-
-        /**
-         * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization,
-         * or commit log has been stopped.
-         */
-        FAILED_TO_SAVE,
-
-        /**
-         * Entry has been successfully evicted, but there were transient listeners present, so we kept the
-         * Node around (transient listeners must survive cache eviction).
-         */
-        EVICTED,
-        ;
-
-        boolean isLoaded()
-        {
-            return this == LOADED || this == MODIFIED || this == FAILED_TO_SAVE;
-        }
-
-        boolean isComplete()
-        {
-            return !(this == LOADING || this == SAVING);
-        }
-    }
-
-    interface State<K, V>
-    {
-        Status status();
-
-        default boolean isCompleteable()
-        {
-            return false;
-        }
-
-        default State<K, V> complete()
-        {
-            throw illegalState(this, "complete()");
-        }
-
-        default Loading<K, V> load(K key, Function<K, V> loadFunction)
-        {
-            throw illegalState(this, "load(key, loadFunction)");
-        }
-
-        default Loaded<K, V> initialize(V value)
-        {
-            throw illegalState(this, "initialize(value)");
-        }
-
-        default RunnableResult<V> loading()
-        {
-            throw illegalState(this, "loading()");
-        }
-
-        default V get()
-        {
-            throw illegalState(this, "get()");
-        }
-
-        default State<K, V> set(V value)
-        {
-            throw illegalState(this, "set(value)");
-        }
-
-        default State<K, V> save(Function<V, Runnable> saveFunction)
-        {
-            throw illegalState(this, "save(saveFunction)");
-        }
-
-        default RunnableResult<Void> saving()
-        {
-            throw illegalState(this, "saving()");
-        }
-
-        default Throwable failure()
-        {
-            throw illegalState(this, "failure()");
-        }
-
-        default Uninitialized<K, V> reset()
-        {
-            throw illegalState(this, "reset()");
-        }
-
-        default Evicted<K, V> evict()
-        {
-            throw illegalState(this, "evict()");
-        }
-
-        default long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return 0;
-        }
-    }
-
-    private static IllegalStateException illegalState(State<?, ?> state, String method)
-    {
-        return new IllegalStateException(format("%s invoked on %s", method, state.status()));
-    }
-
-    static class Uninitialized<K, V> implements State<K, V>
-    {
-        static final Uninitialized<?, ?> instance = new Uninitialized<>();
-
-        @SuppressWarnings("unchecked")
-        static <K, V> Uninitialized<K, V> instance()
-        {
-            return (Uninitialized<K, V>) instance;
-        }
-
-        @Override
-        public Status status()
-        {
-            return UNINITIALIZED;
-        }
-
-        @Override
-        public Loading<K, V> load(K key, Function<K, V> loadFunction)
-        {
-            return new Loading<>(() -> loadFunction.apply(key));
-        }
-
-        public Loaded<K, V> initialize(V value)
-        {
-            return new Loaded<>(value);
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-    }
-
-    static class Loading<K, V> extends RunnableResult<V> implements State<K, V>
-    {
-        Loading(Callable<V> callable)
-        {
-            super(callable);
-        }
-
-        @Override
-        public Status status()
-        {
-            return LOADING;
-        }
-
-        @Override
-        public boolean isCompleteable()
-        {
-            return isDone();
-        }
-
-        @Override
-        public State<K, V> complete()
-        {
-            if      (!isDone())   return this;
-            else if (isSuccess()) return new Loaded<>(result());
-            else                  return new FailedToLoad<>(failure());
-        }
-
-        @Override
-        public RunnableResult<V> loading()
-        {
-            return this;
-        }
-    }
-
-    static class Loaded<K, V> implements State<K, V>
-    {
-        final V original;
-
-        Loaded(V original)
-        {
-            this.original = original;
-        }
-
-        @Override
-        public Status status()
-        {
-            return LOADED;
-        }
-
-        @Override
-        public V get()
-        {
-            return original;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            return value == original ? this : new Modified<>(value);
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-
-        @Override
-        public long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return null == original ? 0 : estimateFunction.applyAsLong(original);
-        }
-    }
-
-    static class FailedToLoad<K, V> implements State<K, V>
-    {
-        final Throwable cause;
-
-        FailedToLoad(Throwable cause)
-        {
-            this.cause = cause;
-        }
-
-        @Override
-        public Status status()
-        {
-            return FAILED_TO_LOAD;
-        }
-
-        @Override
-        public Throwable failure()
-        {
-            return cause;
-        }
-
-        @Override
-        public Uninitialized<K, V> reset()
-        {
-            return Uninitialized.instance();
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-    }
-
-    static class Modified<K, V> implements State<K, V>
-    {
-        V current;
-
-        Modified(V current)
-        {
-            this.current = current;
-        }
-
-        @Override
-        public Status status()
-        {
-            return MODIFIED;
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            current = value;
-            return this;
-        }
-
-        @Override
-        public State<K, V> save(Function<V, Runnable> saveFunction)
-        {
-            Runnable runnable = saveFunction.apply(current);
-            if (null == runnable) // null mutation -> null Runnable -> no change on disk
-                return new Loaded<>(current);
-            else
-                return new Saving<>(current, runnable);
-        }
-
-        @Override
-        public long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return (null == current  ? 0 : estimateFunction.applyAsLong(current));
-        }
-    }
-
-    static class Saving<K, V> extends RunnableResult<Void> implements State<K, V>
-    {
-        V current;
-
-        Saving(V current, Runnable saveRunnable)
-        {
-            this(current, () -> { saveRunnable.run(); return null; });
-        }
-
-        Saving(V current, Callable<Void> saveCallable)
-        {
-            super(saveCallable);
-            this.current = current;
-        }
-
-        @Override
-        public Status status()
-        {
-            return SAVING;
-        }
-
-        @Override
-        public boolean isCompleteable()
-        {
-            return isDone();
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> complete()
-        {
-            if      (!isDone())   return this;
-            else if (isSuccess()) return new Loaded<>(current);
-            else                  return new FailedToSave<>(current, failure());
-        }
-
-        @Override
-        public RunnableResult<Void> saving()
-        {
-            return this;
-        }
-    }
-
-    static class FailedToSave<K, V> implements State<K, V>
-    {
-        V current;
-        final Throwable cause;
-
-        FailedToSave(V current, Throwable cause)
-        {
-            this.current = current;
-            this.cause = cause;
-        }
-
-        @Override
-        public Status status()
-        {
-            return FAILED_TO_SAVE;
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            current = value;
-            return this;
-        }
-
-        @Override
-        public Throwable failure()
-        {
-            return cause;
-        }
-    }
-
-    static class Evicted<K, V> implements State<K, V>
-    {
-        static final Evicted<?, ?> instance = new Evicted<>();
-
-        @SuppressWarnings("unchecked")
-        static <K, V> Evicted<K, V> instance()
-        {
-            return (Evicted<K, V>) instance;
-        }
-
-        @Override
-        public Status status()
-        {
-            return EVICTED;
-        }
-
-        @Override
-        public Uninitialized<K, V> reset()
-        {
-            return Uninitialized.instance();
-        }
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordCommandStore.java

@@ -18,24 +18,20 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.IdentityHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Objects;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntFunction;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import accord.api.Agent;
 import accord.api.DataStore;
@@ -55,6 +51,7 @@ import accord.local.RedundantBefore;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.cfk.CommandsForKey;
+import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
 import accord.primitives.RangeDeps;
 import accord.primitives.Ranges;
@@ -66,22 +63,17 @@ import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
-import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.SequentialExecutorPlus;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.service.accord.SavedCommand.MinimalCommand;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.service.accord.async.AsyncOperation;
-import org.apache.cassandra.service.accord.events.CacheEvents;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Promise;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
-import static accord.local.KeyHistory.COMMANDS;
 import static accord.primitives.SaveStatus.Applying;
+import static accord.local.KeyHistory.SYNC;
 import static accord.primitives.Status.Committed;
 import static accord.primitives.Status.Invalidated;
 import static accord.primitives.Status.Truncated;
@@ -90,87 +82,70 @@ import static org.apache.cassandra.service.accord.SavedCommand.Load.MINIMAL;
 
 public class AccordCommandStore extends CommandStore
 {
-    private static final Logger logger = LoggerFactory.getLogger(AccordCommandStore.class);
-    private static final boolean CHECK_THREADS = CassandraRelevantProperties.TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED.getBoolean();
+    // TODO (required): track this via a PhantomReference, so that if we remove a CommandStore without clearing the caches we can be sure to release them
+    public static class Caches
+    {
+        private final AccordCache global;
+        private final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands;
+        private final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys;
+        private final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys;
+
+        Caches(AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commandCache, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeyCache, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeyCache)
+        {
+            this.global = global;
+            this.commands = commandCache;
+            this.timestampsForKeys = timestampsForKeyCache;
+            this.commandsForKeys = commandsForKeyCache;
+        }
+
+        public final AccordCache global()
+        {
+            return global;
+        }
+
+        public final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands()
+        {
+            return commands;
+        }
+
+        public final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys()
+        {
+            return timestampsForKeys;
+        }
+
+        public final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys()
+        {
+            return commandsForKeys;
+        }
+    }
+
+    public static final class ExclusiveCaches extends Caches implements AutoCloseable
+    {
+        private final Lock lock;
+
+        public ExclusiveCaches(Lock lock, AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys)
+        {
+            super(global, commands, timestampsForKeys, commandsForKeys);
+            this.lock = lock;
+        }
+
+        @Override
+        public void close()
+        {
+            lock.unlock();
+        }
+    }
 
     public final String loggingId;
     private final IJournal journal;
-
-    private final CommandStoreExecutor executor;
-    private final AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache;
-    private final AccordStateCache.Instance<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache;
-    private final AccordStateCache.Instance<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache;
-    private AsyncOperation<?> currentOperation = null;
-    private AccordSafeCommandStore current = null;
+    private final AccordExecutor executor;
+    private final Executor taskExecutor;
+    private final ExclusiveCaches caches;
     private long lastSystemTimestampMicros = Long.MIN_VALUE;
     private final CommandsForRangesLoader commandsForRangesLoader;
 
-    private static <K, V> void registerJfrListener(int id, AccordStateCache.Instance<K, V, ?> instance, String name)
-    {
-        if (!DatabaseDescriptor.getAccordStateCacheListenerJFREnabled())
-            return;
-        instance.register(new AccordStateCache.Listener<>() {
-            private final IdentityHashMap<AccordCachingState<?, ?>, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>();
-
-            @Override
-            public void onAdd(AccordCachingState<K, V> state)
-            {
-                CacheEvents.Add add = new CacheEvents.Add();
-                CacheEvents.Evict evict = new CacheEvents.Evict();
-                if (!add.isEnabled())
-                    return;
-                add.begin();
-                evict.begin();
-                add.store = evict.store = id;
-                add.instance = evict.instance = name;
-                add.key = evict.key = state.key().toString();
-                updateMutable(instance, state, add);
-                add.commit();
-                pendingEvicts.put(state, evict);
-            }
-
-            @Override
-            public void onRelease(AccordCachingState<K, V> state)
-            {
-
-            }
-
-            @Override
-            public void onEvict(AccordCachingState<K, V> state)
-            {
-                CacheEvents.Evict event = pendingEvicts.remove(state);
-                if (event == null) return;
-                updateMutable(instance, state, event);
-                event.commit();
-            }
-        });
-    }
-
-    private static void updateMutable(AccordStateCache.Instance<?, ?, ?> instance, AccordCachingState<?, ?> state, CacheEvents event)
-    {
-        event.status = state.state().status().name();
-
-        event.lastQueriedEstimatedSizeOnHeap = state.lastQueriedEstimatedSizeOnHeap();
-
-        event.instanceAllocated = instance.weightedSize();
-        AccordStateCache.Stats stats = instance.stats();
-        event.instanceStatsQueries = stats.queries;
-        event.instanceStatsHits = stats.hits;
-        event.instanceStatsMisses = stats.misses;
-
-        event.globalSize = instance.size();
-        event.globalReferenced = instance.globalReferencedEntries();
-        event.globalUnreferenced = instance.globalUnreferencedEntries();
-        event.globalCapacity = instance.capacity();
-        event.globalAllocated = instance.globalAllocated();
-
-        stats = instance.globalStats();
-        event.globalStatsQueries = stats.queries;
-        event.globalStatsHits = stats.hits;
-        event.globalStatsMisses = stats.misses;
-
-        event.update();
-    }
+    private AccordSafeCommandStore current;
+    private Thread currentThread;
 
     public AccordCommandStore(int id,
                               NodeCommandStoreService node,
@@ -180,53 +155,33 @@ public class AccordCommandStore extends CommandStore
                               LocalListeners.Factory listenerFactory,
                               EpochUpdateHolder epochUpdateHolder,
                               IJournal journal,
-                              CommandStoreExecutor commandStoreExecutor)
+                              AccordExecutor executor)
     {
         super(id, node, agent, dataStore, progressLogFactory, listenerFactory, epochUpdateHolder);
-        this.journal = journal;
         loggingId = String.format("[%s]", id);
-        executor = commandStoreExecutor;
-        AccordStateCache stateCache = executor.stateCache;
-        commandCache =
-            stateCache.instance(TxnId.class,
-                                AccordSafeCommand.class,
-                                AccordSafeCommand.safeRefFactory(),
-                                this::loadCommand,
-                                this::appendToKeyspace,
-                                this::validateCommand,
-                                AccordObjectSizes::command);
-        registerJfrListener(id, commandCache, "Command");
-        timestampsForKeyCache =
-            stateCache.instance(RoutingKey.class,
-                                AccordSafeTimestampsForKey.class,
-                                AccordSafeTimestampsForKey::new,
-                                this::loadTimestampsForKey,
-                                this::saveTimestampsForKey,
-                                this::validateTimestampsForKey,
-                                AccordObjectSizes::timestampsForKey);
-        registerJfrListener(id, timestampsForKeyCache, "TimestampsForKey");
-        commandsForKeyCache =
-            stateCache.instance(RoutingKey.class,
-                                AccordSafeCommandsForKey.class,
-                                AccordSafeCommandsForKey::new,
-                                this::loadCommandsForKey,
-                                this::saveCommandsForKey,
-                                this::validateCommandsForKey,
-                                AccordObjectSizes::commandsForKey,
-                                AccordCachingState::new);
-        registerJfrListener(id, commandsForKeyCache, "CommandsForKey");
+        this.journal = journal;
+        this.executor = executor;
 
+        final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands;
+        final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKey;
+        final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKey;
+        try (AccordExecutor.ExclusiveGlobalCaches exclusive = executor.lockCaches())
+        {
+            commands = exclusive.commands.newInstance(this);
+            timestampsForKey = exclusive.timestampsForKey.newInstance(this);
+            commandsForKey = exclusive.commandsForKey.newInstance(this);
+            this.caches = new ExclusiveCaches(executor.lock, exclusive.global, commands, timestampsForKey, commandsForKey);
+        }
+
+        this.taskExecutor = executor.executor(this);
         this.commandsForRangesLoader = new CommandsForRangesLoader(this);
-
         loadRedundantBefore(journal.loadRedundantBefore(id()));
         loadBootstrapBeganAt(journal.loadBootstrapBeganAt(id()));
         loadSafeToRead(journal.loadSafeToRead(id()));
         loadRangesForEpoch(journal.loadRangesForEpoch(id()));
-
-        executor.execute(() -> CommandStore.register(this));
     }
 
-    static Factory factory(AccordJournal journal, IntFunction<CommandStoreExecutor> executorFactory)
+    static Factory factory(AccordJournal journal, IntFunction<AccordExecutor> executorFactory)
     {
         return (id, node, agent, dataStore, progressLogFactory, listenerFactory, rangesForEpoch) ->
                new AccordCommandStore(id, node, agent, dataStore, progressLogFactory, listenerFactory, rangesForEpoch, journal, executorFactory.apply(id));
@@ -247,61 +202,65 @@ public class AccordCommandStore extends CommandStore
     @Override
     public boolean inStore()
     {
-        return executor.isInThread();
+        return currentThread == Thread.currentThread();
     }
 
-    public void checkInStoreThread()
+    void tryPreSetup(AccordTask<?> task)
     {
-        checkState(inStore());
+        if (inStore() && current != null)
+            task.presetup(current.task);
     }
 
-    public void checkNotInStoreThread()
+    public AccordExecutor executor()
     {
-        if (!CHECK_THREADS)
-            return;
-        checkState(!inStore());
+        return executor;
     }
 
-    public ExecutorService executor()
+    // TODO (desired): we use this for executing callbacks with mutual exclusivity,
+    //  but we don't need to block the actual CommandStore - could quite easily
+    //  inflate a separate queue dynamically in AccordExecutor
+    public Executor taskExecutor()
     {
-        return executor.delegate();
+        return taskExecutor;
     }
 
-    /**
-     * Note that this cache is shared with other commandStores!
-     */
-    public AccordStateCache cache()
+    public ExclusiveCaches lockCaches()
     {
-        return executor.cache();
+        //noinspection LockAcquiredButNotSafelyReleased
+        caches.lock.lock();
+        return caches;
     }
 
-    public AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache()
+    public ExclusiveCaches tryLockCaches()
     {
-        return commandCache;
+        if (caches.lock.tryLock())
+            return caches;
+        return null;
     }
 
-    public AccordStateCache.Instance<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache()
+    public Caches cachesExclusive()
     {
-        return timestampsForKeyCache;
+        Invariants.checkState(executor.isOwningThread());
+        return caches;
     }
 
-    public AccordStateCache.Instance<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache()
+    public Caches cachesUnsafe()
     {
-        return commandsForKeyCache;
+        return caches;
     }
 
     @VisibleForTesting
     @Override
-    protected void unsafeSetRangesForEpoch(CommandStores.RangesForEpoch newRangesForEpoch)
+    public void unsafeSetRangesForEpoch(CommandStores.RangesForEpoch newRangesForEpoch)
     {
         super.unsafeSetRangesForEpoch(newRangesForEpoch);
     }
 
     @Nullable
     @VisibleForTesting
-    public Runnable appendToKeyspace(Command after)
+    public Runnable appendToKeyspace(TxnId txnId, Command after)
     {
-        if (after.txnId().is(Routable.Domain.Key))
+        if (txnId.is(Routable.Domain.Key))
             return null;
 
         Mutation mutation = AccordKeyspace.getCommandMutation(this.id, after, nextSystemTimestampMicros());
@@ -345,18 +304,18 @@ public class AccordCommandStore extends CommandStore
         if (!Invariants.isParanoid())
             return true;
 
-        TimestampsForKey reloaded = AccordKeyspace.unsafeLoadTimestampsForKey(this, (TokenKey) key);
+        TimestampsForKey reloaded = AccordKeyspace.unsafeLoadTimestampsForKey(id, (TokenKey) key);
         return Objects.equals(evicting, reloaded);
     }
 
     TimestampsForKey loadTimestampsForKey(RoutableKey key)
     {
-        return AccordKeyspace.loadTimestampsForKey(this, (TokenKey) key);
+        return AccordKeyspace.loadTimestampsForKey(id, (TokenKey) key);
     }
 
     CommandsForKey loadCommandsForKey(RoutableKey key)
     {
-        return AccordKeyspace.loadCommandsForKey(this, (TokenKey) key);
+        return AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
     }
 
     boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting)
@@ -364,40 +323,20 @@ public class AccordCommandStore extends CommandStore
         if (!Invariants.isParanoid())
             return true;
 
-        CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(this, (TokenKey) key);
+        CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
         return Objects.equals(evicting, reloaded);
     }
 
     @Nullable
-    private Runnable saveTimestampsForKey(TimestampsForKey after)
+    Runnable saveTimestampsForKey(RoutingKey key, TimestampsForKey after, Object serialized)
     {
-        Mutation mutation = AccordKeyspace.getTimestampsForKeyMutation(id, after, nextSystemTimestampMicros());
-        return null != mutation ? mutation::applyUnsafe : null;
+        return AccordKeyspace.getTimestampsForKeyUpdater(this, after, nextSystemTimestampMicros());
     }
 
     @Nullable
-    private Runnable saveCommandsForKey(CommandsForKey after)
+    Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized)
     {
-        Mutation mutation = AccordKeyspace.getCommandsForKeyMutation(id, after, nextSystemTimestampMicros());
-        return null != mutation ? mutation::applyUnsafe : null;
-    }
-
-    public void setCurrentOperation(AsyncOperation<?> operation)
-    {
-        checkState(currentOperation == null);
-        currentOperation = operation;
-    }
-
-    public AsyncOperation<?> getContext()
-    {
-        checkState(currentOperation != null);
-        return currentOperation;
-    }
-
-    public void unsetCurrentOperation(AsyncOperation<?> operation)
-    {
-        checkState(currentOperation == operation);
-        currentOperation = null;
+        return AccordKeyspace.getCommandsForKeyUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros());
     }
 
     public long nextSystemTimestampMicros()
@@ -408,13 +347,13 @@ public class AccordCommandStore extends CommandStore
     @Override
     public <T> AsyncChain<T> submit(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
     {
-        return AsyncOperation.create(this, loadCtx, function);
+        return AccordTask.create(this, loadCtx, function).chain();
     }
 
     @Override
     public <T> AsyncChain<T> submit(Callable<T> task)
     {
-        return AsyncChains.ofCallable(executor.delegate(), task);
+        return AsyncChains.ofCallable(taskExecutor(), task);
     }
 
     public DataStore dataStore()
@@ -435,7 +374,7 @@ public class AccordCommandStore extends CommandStore
     @Override
     public AsyncChain<Void> execute(PreLoadContext preLoadContext, Consumer<? super SafeCommandStore> consumer)
     {
-        return AsyncOperation.create(this, preLoadContext, consumer);
+        return AccordTask.create(this, preLoadContext, consumer).chain();
     }
 
     public void executeBlocking(Runnable runnable)
@@ -454,43 +393,38 @@ public class AccordCommandStore extends CommandStore
         }
     }
 
-    public AccordSafeCommandStore beginOperation(PreLoadContext preLoadContext,
-                                                 Map<TxnId, AccordSafeCommand> commands,
-                                                 Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKeys,
-                                                 Map<RoutingKey, AccordSafeCommandsForKey> commandsForKeys,
-                                                 @Nullable AccordSafeCommandsForRanges commandsForRanges)
+    public AccordSafeCommandStore begin(AccordTask<?> operation,
+                                        @Nullable CommandsForRanges commandsForRanges)
     {
         checkState(current == null);
-        commands.values().forEach(AccordSafeState::preExecute);
-        commandsForKeys.values().forEach(AccordSafeState::preExecute);
-        timestampsForKeys.values().forEach(AccordSafeState::preExecute);
-        if (commandsForRanges != null)
-            commandsForRanges.preExecute();
-
-        current = AccordSafeCommandStore.create(preLoadContext, commands, timestampsForKeys, commandsForKeys, commandsForRanges, this);
+        current = AccordSafeCommandStore.create(operation, commandsForRanges, this);
         return current;
     }
 
+    void setOwner(Thread thread, Thread self)
+    {
+        Invariants.checkState(thread == null ? currentThread == self : currentThread == null);
+        currentThread = thread;
+        if (thread != null) CommandStore.register(this);
+
+    }
+
     public boolean hasSafeStore()
     {
         return current != null;
     }
 
-    public void completeOperation(AccordSafeCommandStore store)
+    public void complete(AccordSafeCommandStore store)
     {
         checkState(current == store);
-        try
-        {
-            current.postExecute();
-        }
-        finally
-        {
-            current = null;
-        }
+        current.postExecute();
+        current = null;
     }
 
-    public void abortCurrentOperation()
+    public void abort(AccordSafeCommandStore store)
     {
+        checkInStore();
+        Invariants.checkState(store == current);
         current = null;
     }
 
@@ -511,34 +445,37 @@ public class AccordCommandStore extends CommandStore
         Ranges allRanges = safeStore.ranges().all();
         Ranges coordinateRanges = Ranges.EMPTY;
         long coordinateEpoch = -1;
-        for (int i = 0; i < rangeDeps.txnIdCount(); i++)
+        try (ExclusiveCaches caches = lockCaches())
         {
-            TxnId txnId = rangeDeps.txnId(i);
-            AccordCachingState<TxnId, Command> state = commandCache.getUnsafe(txnId);
-            if (state != null && state.isLoaded() && state.get() != null && state.get().known().isDefinitionKnown())
-                continue;
-
-            Ranges addRanges = rangeDeps.ranges(i).slice(allRanges);
-            if (addRanges.isEmpty()) continue;
-
-            if (coordinateEpoch != txnId.epoch())
+            for (int i = 0; i < rangeDeps.txnIdCount(); i++)
             {
-                coordinateEpoch = txnId.epoch();
-                coordinateRanges = ranges.allAt(txnId.epoch());
+                TxnId txnId = rangeDeps.txnId(i);
+                AccordCacheEntry<TxnId, Command> state = caches.commands().getUnsafe(txnId);
+                if (state != null && state.isLoaded() && state.getExclusive() != null && state.getExclusive().known().isDefinitionKnown())
+                    continue;
+
+                Ranges addRanges = rangeDeps.ranges(i).slice(allRanges);
+                if (addRanges.isEmpty()) continue;
+
+                if (coordinateEpoch != txnId.epoch())
+                {
+                    coordinateEpoch = txnId.epoch();
+                    coordinateRanges = ranges.allAt(txnId.epoch());
+                }
+                if (addRanges.intersects(coordinateRanges)) continue;
+                addRanges = redundantBefore.removeShardRedundant(txnId, txnId, addRanges);
+                if (addRanges.isEmpty()) continue;
+                diskCommandsForRanges().mergeTransitive(txnId, addRanges, Ranges::with);
             }
-            if (addRanges.intersects(coordinateRanges)) continue;
-            addRanges = redundantBefore.removeShardRedundant(txnId, txnId, addRanges);
-            if (addRanges.isEmpty()) continue;
-            diskCommandsForRanges().mergeTransitive(txnId, addRanges, Ranges::with);
         }
     }
 
-    public void appendCommands(List<SavedCommand.DiffWriter> diffs, Runnable onFlush)
+    public void appendCommands(List<SavedCommand.Writer> diffs, Runnable onFlush)
     {
         for (int i = 0; i < diffs.size(); i++)
         {
             boolean isLast = i == diffs.size() - 1;
-            SavedCommand.DiffWriter writer = diffs.get(i);
+            SavedCommand.Writer writer = diffs.get(i);
             journal.appendCommand(id, writer, isLast  ? onFlush : null);
         }
     }
@@ -549,6 +486,21 @@ public class AccordCommandStore extends CommandStore
         return journal.loadCommand(id, txnId, unsafeGetRedundantBefore(), durableBefore());
     }
 
+    public static Command prepareToCache(Command command)
+    {
+        // TODO (required): validate we don't have duplicate objects
+        if (command != null)
+        {
+            PartialTxn txn = command.partialTxn();
+            if (txn != null)
+            {
+                TxnRead read = (TxnRead) txn.read();
+                read.unmemoize();
+            }
+        }
+        return command;
+    }
+
     public MinimalCommand loadMinimal(TxnId txnId)
     {
         return journal.loadMinimal(id, txnId, MINIMAL, unsafeGetRedundantBefore(), durableBefore());
@@ -568,22 +520,13 @@ public class AccordCommandStore extends CommandStore
             {
                 TxnId txnId = command.txnId();
                 Participants<?> keys = null;
-                List<TxnId> deps = null;
                 if (CommandsForKey.manages(txnId))
                     keys = command.hasBeen(Committed) ? command.participants().hasTouched() : command.participants().touches();
                 else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
                     keys = command.asCommitted().waitingOn.keys;
 
-                if (command.partialDeps() != null)
-                    deps = command.partialDeps().txnIds();
-
                 if (keys != null)
-                {
-                    if (deps != null)
-                        return PreLoadContext.contextFor(txnId, deps, keys, keyHistory);
-
                     return PreLoadContext.contextFor(txnId, keys, keyHistory);
-                }
 
                 return PreLoadContext.contextFor(txnId);
             }
@@ -593,12 +536,12 @@ public class AccordCommandStore extends CommandStore
                 TxnId txnId = command.txnId();
 
                 AsyncPromise<?> future = new AsyncPromise<>();
-                execute(context(command, COMMANDS),
+                execute(context(command, SYNC),
                         safeStore -> {
                             Command local = command;
                             if (local.status() != Truncated && local.status() != Invalidated)
                             {
-                                Cleanup cleanup = Cleanup.shouldCleanup(local, unsafeGetRedundantBefore(), durableBefore());
+                                Cleanup cleanup = Cleanup.shouldCleanup(agent, local, unsafeGetRedundantBefore(), durableBefore());
                                 switch (cleanup)
                                 {
                                     case NO:
@@ -678,116 +621,4 @@ public class AccordCommandStore extends CommandStore
         if (rangesForEpoch != null)
             unsafeSetRangesForEpoch(new CommandStores.RangesForEpoch(rangesForEpoch.epochs, rangesForEpoch.ranges, this));
     }
-
-    public static class CommandStoreExecutor implements CacheSize
-    {
-        final AccordStateCache stateCache;
-        final SequentialExecutorPlus delegate;
-        final long threadId;
-
-        CommandStoreExecutor(AccordStateCache stateCache, SequentialExecutorPlus delegate, long threadId)
-        {
-            this.stateCache = stateCache;
-            this.delegate = delegate;
-            this.threadId = threadId;
-        }
-
-        public boolean hasTasks()
-        {
-            return delegate.getPendingTaskCount() > 0 || delegate.getActiveTaskCount() > 0;
-        }
-
-        CommandStoreExecutor(AccordStateCache stateCache, SequentialExecutorPlus delegate)
-        {
-            this.stateCache = stateCache;
-            this.delegate = delegate;
-            this.threadId = getThreadId();
-        }
-
-        public boolean isInThread()
-        {
-            if (!CHECK_THREADS)
-                return true;
-
-            return threadId == Thread.currentThread().getId();
-        }
-
-        public void shutdown()
-        {
-            delegate.shutdown();
-        }
-
-        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
-        {
-            return delegate.awaitTermination(timeout, unit);
-        }
-
-        public Future<?> submit(Runnable task)
-        {
-            return delegate.submit(task);
-        }
-
-        public ExecutorService delegate()
-        {
-            return delegate;
-        }
-
-        public void execute(Runnable command)
-        {
-            delegate.submit(command);
-        }
-
-        private long getThreadId()
-        {
-            try
-            {
-                return delegate.submit(() -> Thread.currentThread().getId()).get();
-            }
-            catch (InterruptedException e)
-            {
-                throw new AssertionError(e);
-            }
-            catch (ExecutionException e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @VisibleForTesting
-        public AccordStateCache cache()
-        {
-            return stateCache;
-        }
-
-        @VisibleForTesting
-        public void unsafeClearCache()
-        {
-            stateCache.unsafeClear();
-        }
-
-        @Override
-        public void setCapacity(long bytes)
-        {
-            Invariants.checkState(isInThread());
-            stateCache.setCapacity(bytes);
-        }
-
-        @Override
-        public long capacity()
-        {
-            return stateCache.capacity();
-        }
-
-        @Override
-        public int size()
-        {
-            return stateCache.size();
-        }
-
-        @Override
-        public long weightedSize()
-        {
-            return stateCache.weightedSize();
-        }
-    }
 }

src/java/org/apache/cassandra/service/accord/AccordCommandStores.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -27,7 +28,6 @@ import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.LocalListeners;
 import accord.api.ProgressLog;
-import accord.local.CommandStore;
 import accord.local.CommandStores;
 import accord.local.Node;
 import accord.local.NodeCommandStoreService;
@@ -36,50 +36,87 @@ import accord.primitives.Range;
 import accord.topology.Topology;
 import accord.utils.RandomSource;
 import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.AccordSpec.QueueShardModel;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
 import org.apache.cassandra.metrics.CacheSizeMetrics;
 import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.accord.AccordCommandStore.CommandStoreExecutor;
+import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.concurrent.Stage.ACCORD_MIGRATION;
-import static org.apache.cassandra.concurrent.Stage.ACCORD_RANGE_LOADER;
-import static org.apache.cassandra.concurrent.Stage.MUTATION;
-import static org.apache.cassandra.concurrent.Stage.READ;
+import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_PER_SHARD;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordQueueSubmissionModel;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordQueueShardCount;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+import static org.apache.cassandra.service.accord.AccordExecutor.constant;
+import static org.apache.cassandra.service.accord.AccordExecutor.constantFactory;
 
 public class AccordCommandStores extends CommandStores implements CacheSize
 {
     public static final String ACCORD_STATE_CACHE = "AccordStateCache";
 
     private final CacheSizeMetrics cacheSizeMetrics;
-    private final CommandStoreExecutor[] executors;
-    private long cacheSize;
+    private final AccordExecutor[] executors;
+    private long cacheSize, workingSetSize;
+    private int maxQueuedLoads, maxQueuedRangeLoads;
+    private boolean shrinkingOn;
 
     AccordCommandStores(NodeCommandStoreService node, Agent agent, DataStore store, RandomSource random,
                         ShardDistributor shardDistributor, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenerFactory,
-                        AccordJournal journal, CommandStoreExecutor[] executors)
+                        AccordJournal journal, AccordExecutor[] executors)
     {
         super(node, agent, store, random, shardDistributor, progressLogFactory, listenerFactory,
               AccordCommandStore.factory(journal, id -> executors[id % executors.length]));
-        setCapacity(DatabaseDescriptor.getAccordCacheSizeInMiB() << 20);
         this.executors = executors;
         this.cacheSizeMetrics = new CacheSizeMetrics(ACCORD_STATE_CACHE, this);
+        cacheSize = DatabaseDescriptor.getAccordCacheSizeInMiB() << 20;
+        workingSetSize = DatabaseDescriptor.getAccordWorkingSetSizeInMiB() << 20;
+        maxQueuedLoads = DatabaseDescriptor.getAccordMaxQueuedLoadCount();
+        maxQueuedRangeLoads = DatabaseDescriptor.getAccordMaxQueuedRangeLoadCount();
+        shrinkingOn = DatabaseDescriptor.getAccordCacheShrinkingOn();
+        refreshCapacities();
     }
 
     static Factory factory(AccordJournal journal)
     {
         return (time, agent, store, random, shardDistributor, progressLogFactory, listenerFactory) -> {
-            CommandStoreExecutor[] executors = new CommandStoreExecutor[DatabaseDescriptor.getAccordShardCount()];
+            AccordExecutor[] executors = new AccordExecutor[getAccordQueueShardCount()];
+            AccordExecutorFactory factory;
+            int maxThreads = Integer.MAX_VALUE;
+            switch (getAccordQueueSubmissionModel())
+            {
+                default: throw new AssertionError("Unhandled QueueSubmissionModel: " + getAccordQueueSubmissionModel());
+                case SYNC: factory = AccordExecutorSyncSubmit::new; break;
+                case SEMI_SYNC: factory = AccordExecutorSemiSyncSubmit::new; break;
+                case ASYNC: factory = AccordExecutorAsyncSubmit::new; break;
+                case EXEC_ST:
+                    factory = AccordExecutorSimple::new;
+                    maxThreads = 1;
+                    break;
+            }
+
             for (int id = 0; id < executors.length; id++)
             {
-                AccordStateCacheMetrics metrics = new AccordStateCacheMetrics(ACCORD_STATE_CACHE);
-                AccordStateCache stateCache = new AccordStateCache(Stage.READ.executor(), Stage.MUTATION.executor(), 8 << 20, metrics);
-                executors[id] = new CommandStoreExecutor(stateCache, executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + id + ']'));
+                AccordCacheMetrics metrics = new AccordCacheMetrics(ACCORD_STATE_CACHE);
+                QueueShardModel shardModel = DatabaseDescriptor.getAccordQueueShardModel();
+                String baseName = AccordExecutor.class.getSimpleName() + '[' + id;
+                int threads = Math.min(maxThreads, Math.max(DatabaseDescriptor.getAccordConcurrentOps() / getAccordQueueShardCount(), 1));
+                switch (shardModel)
+                {
+                    case THREAD_PER_SHARD:
+                    case THREAD_PER_SHARD_SYNC_QUEUE:
+                        executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD ? RUN_WITHOUT_LOCK : RUN_WITH_LOCK, 1, constant(baseName + ']'), metrics, constantFactory(Stage.READ.executor()), constantFactory(Stage.MUTATION.executor()), constantFactory(Stage.READ.executor()), agent);
+                        break;
+                    case THREAD_POOL_PER_SHARD:
+                        executors[id] = factory.get(id, RUN_WITHOUT_LOCK, threads, num -> baseName + ',' + num + ']', metrics, AccordExecutor::submitIOToSelf, AccordExecutor::submitIOToSelf, AccordExecutor::submitIOToSelf, agent);
+                        break;
+                    case THREAD_POOL_PER_SHARD_EXCLUDES_IO:
+                        executors[id] = factory.get(id, RUN_WITHOUT_LOCK, threads, num -> baseName + ',' + num + ']', metrics, constantFactory(Stage.READ.executor()), constantFactory(Stage.MUTATION.executor()), constantFactory(Stage.READ.executor()), agent);
+                        break;
+                }
             }
 
             return new AccordCommandStores(time, agent, store, random, shardDistributor, progressLogFactory, listenerFactory, journal, executors);
@@ -109,7 +146,27 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public synchronized void setCapacity(long bytes)
     {
         cacheSize = bytes;
-        refreshCacheSizes();
+        refreshCapacities();
+    }
+
+    public synchronized void setWorkingSetSize(long bytes)
+    {
+        workingSetSize = bytes;
+        refreshCapacities();
+    }
+
+    public synchronized void setCapacityAndWorkingSetSize(long newCacheSize, long newWorkingSetSize)
+    {
+        cacheSize = newCacheSize;
+        workingSetSize = newWorkingSetSize;
+        refreshCapacities();
+    }
+
+    public synchronized void setMaxQueuedLoads(int total, int range)
+    {
+        maxQueuedLoads = total;
+        maxQueuedRangeLoads = range;
+        refreshCapacities();
     }
 
     @Override
@@ -122,7 +179,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public int size()
     {
         int size = 0;
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
             size += executor.size();
         return size;
     }
@@ -131,19 +188,31 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public long weightedSize()
     {
         long size = 0;
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
             size += executor.weightedSize();
         return size;
     }
 
-    synchronized void refreshCacheSizes()
+    synchronized void refreshCapacities()
     {
-        if (count() == 0)
-            return;
-        long perExecutor = cacheSize / executors.length;
-        // TODO (low priority, safety): we might transiently breach our limit if we increase one store before decreasing another
-        for (CommandStoreExecutor executor : executors)
-            executor.execute(() -> executor.setCapacity(perExecutor));
+        long capacityPerExecutor = cacheSize / executors.length;
+        long workingSetPerExecutor = workingSetSize < 0 ? Long.MAX_VALUE : workingSetSize / executors.length;
+        int maxLoadsPerExecutor = (maxQueuedLoads + executors.length - 1) / executors.length;
+        int maxRangeLoadsPerExecutor = (maxQueuedRangeLoads + executors.length - 1) / executors.length;
+        for (AccordExecutor executor : executors)
+        {
+            executor.executeDirectlyWithLock(() -> {
+                executor.setCapacity(capacityPerExecutor);
+                executor.setWorkingSetSize(workingSetPerExecutor);
+                executor.setMaxQueuedLoads(maxLoadsPerExecutor, maxRangeLoadsPerExecutor);
+                executor.cacheExclusive().setShrinkingOn(shrinkingOn);
+            });
+        }
+    }
+
+    public List<AccordExecutor> executors()
+    {
+        return Arrays.asList(executors.clone());
     }
 
     public void waitForQuiescense()
@@ -151,23 +220,13 @@ public class AccordCommandStores extends CommandStores implements CacheSize
         boolean hadPending;
         try
         {
-            List<ExecutorPlus> executors = new ArrayList<>();
-            for (CommandStoreExecutor executor : this.executors)
-                executors.add(executor.delegate);
-
-            executors.add(READ.executor());
-            executors.add(MUTATION.executor());
-            executors.add(ACCORD_MIGRATION.executor());
-            executors.add(ACCORD_RANGE_LOADER.executor());
-
             do
             {
                 hadPending = false;
                 List<Future<?>> futures = new ArrayList<>();
-                for (ExecutorPlus executor : executors)
+                for (AccordExecutor executor : this.executors)
                 {
-                    if (!hadPending && (executor.getPendingTaskCount() > 0 || executor.getActiveTaskCount() > 0))
-                        hadPending = true;
+                    hadPending |= executor.hasTasks();
                     futures.add(executor.submit(() -> {}));
                 }
                 for (Future<?> future : futures)
@@ -190,7 +249,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public synchronized void shutdown()
     {
         super.shutdown();
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
         {
             executor.shutdown();
             try

src/java/org/apache/cassandra/service/accord/AccordDataStore.java

@@ -38,7 +38,6 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.memtable.Memtable;
@@ -68,7 +67,7 @@ public class AccordDataStore implements DataStore
     static class SnapshotBounds
     {
         final List<org.apache.cassandra.dht.Range<Token>> ranges = new ArrayList<>();
-        CommitLogPosition position;
+        long id;
     }
 
     @Override
@@ -93,7 +92,7 @@ public class AccordDataStore implements DataStore
             ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata);
             // TODO (required): when we can safely map TxnId.hlc() -> local timestamp, consult Memtable timestamps
             Memtable memtable = cfs.getCurrentMemtable();
-            e.getValue().position = memtable.getCommitLogLowerBound();
+            e.getValue().id = memtable.getMemtableId();
         }
 
         ScheduledExecutors.scheduledTasks.schedule(() -> {
@@ -106,7 +105,7 @@ public class AccordDataStore implements DataStore
                 View view = cfs.getTracker().getView();
                 for (Memtable memtable : view.getAllMemtables())
                 {
-                    if (memtable.getCommitLogLowerBound().compareTo(bounds.position) > 0) continue;
+                    if (memtable.getMemtableId() > bounds.id) continue;
                     if (!intersects(cfs, memtable, bounds.ranges)) continue;
 
                     futures.add(cfs.forceFlush(ACCORD_TXN_GC));

src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java

@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.locks.Lock;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.utils.concurrent.ConcurrentLinkedStack;
+
+import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+
+abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
+{
+    final ConcurrentLinkedStack<Object> submitted = new ConcurrentLinkedStack<>();
+    boolean isHeldByExecutor;
+
+    AccordExecutorAbstractLockLoop(Lock lock, int executorId, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    abstract void notifyWorkExclusive();
+    abstract void awaitExclusive() throws InterruptedException;
+    abstract boolean isInLoop();
+    abstract <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4);
+
+    <P1s, P1a, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        // if we're a loop thread, we will poll the waitingToRun queue when we come around
+        if (isInLoop()) submitted.push(async.apply(p1a, p2, p3, p4));
+        else submitExternal(sync, async, p1s, p1a, p2, p3, p4);
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternalExclusive(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        try
+        {
+            try
+            {
+                drainSubmittedExclusive();
+            }
+            catch (Throwable t)
+            {
+                try { sync.accept(this, p1s, p2, p3, p4); }
+                catch (Throwable t2) { t.addSuppressed(t2); }
+                throw t;
+            }
+            sync.accept(this, p1s, p2, p3, p4);
+        }
+        finally
+        {
+            notifyIfMoreWorkExclusive();
+        }
+    }
+
+    public boolean hasTasks()
+    {
+        if (tasks > 0 || !submitted.isEmpty() || running > 0)
+            return true;
+
+        lock.lock();
+        try
+        {
+            return tasks > 0 || !submitted.isEmpty() || running > 0;
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    void updateWaitingToRunExclusive()
+    {
+        drainSubmittedExclusive();
+        super.updateWaitingToRunExclusive();
+    }
+
+    void drainSubmittedExclusive()
+    {
+        submitted.drain(AccordExecutor::consumeExclusive, this, true);
+    }
+
+    void notifyIfMoreWorkExclusive()
+    {
+        if (hasWaitingToRun())
+            notifyWorkExclusive();
+    }
+
+    private void enterLockExclusive()
+    {
+        isHeldByExecutor = true;
+    }
+
+    private void exitLockExclusive()
+    {
+        isHeldByExecutor = false;
+        notifyIfMoreWorkExclusive();
+    }
+
+    private void pauseExclusive()
+    {
+        --running;
+    }
+
+    private void resumeExclusive()
+    {
+        ++running;
+    }
+
+    Interruptible.Task task(Mode mode)
+    {
+        return mode == RUN_WITH_LOCK ? this::runWithLock : this::runWithoutLock;
+    }
+
+    protected void runWithLock(Interruptible.State state) throws InterruptedException
+    {
+        lock.lockInterruptibly();
+        try
+        {
+            resumeExclusive();
+            enterLockExclusive();
+            while (true)
+            {
+                Task task = pollWaitingToRunExclusive();
+
+                if (task != null)
+                {
+                    --tasks;
+                    try
+                    {
+                        task.preRunExclusive();
+                        task.run();
+                    }
+                    catch (Throwable t)
+                    {
+                        task.fail(t);
+                    }
+                    finally
+                    {
+                        task.cleanupExclusive();
+                    }
+                }
+                else
+                {
+                    if (state != NORMAL)
+                    {
+                        pauseExclusive();
+                        exitLockExclusive();
+                        return;
+                    }
+
+                    pauseExclusive();
+                    awaitExclusive();
+                    resumeExclusive();
+                }
+            }
+        }
+        catch (Throwable t)
+        {
+            pauseExclusive();
+            exitLockExclusive();
+            throw t;
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    protected void runWithoutLock(Interruptible.State state) throws InterruptedException
+    {
+        Task task = null;
+        while (true)
+        {
+            lock.lock();
+            try
+            {
+                if (task != null) task.cleanupExclusive();
+                else resumeExclusive();
+                enterLockExclusive();
+
+                while (true)
+                {
+                    task = pollWaitingToRunExclusive();
+                    if (task != null)
+                    {
+                        exitLockExclusive();
+                        break;
+                    }
+
+                    if (state != NORMAL)
+                    {
+                        exitLockExclusive();
+                        return;
+                    }
+
+                    pauseExclusive();
+                    awaitExclusive();
+                    resumeExclusive();
+                }
+                --tasks;
+                task.preRunExclusive();
+            }
+            catch (Throwable t)
+            {
+                if (task != null)
+                {
+                    try { task.fail(t); }
+                    catch (Throwable t2) { t.addSuppressed(t2); }
+                    try { task.cleanupExclusive(); }
+                    catch (Throwable t2) { t.addSuppressed(t2); }
+                    try { agent.onUncaughtException(t); }
+                    catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ }
+                }
+                pauseExclusive();
+                exitLockExclusive();
+                throw t;
+            }
+            finally
+            {
+                lock.unlock();
+            }
+
+            try
+            {
+                task.run();
+            }
+            catch (Throwable t)
+            {
+                try { task.fail(t); }
+                catch (Throwable t2)
+                {
+                    try
+                    {
+                        t2.addSuppressed(t);
+                        agent.onUncaughtException(t2);
+                    }
+                    catch (Throwable t3)
+                    {
+                        // empty to ensure we definitely loop so we cleanup the task
+                    }
+                }
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.locks.Lock;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+
+abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstractLockLoop
+{
+    AccordExecutorAbstractSemiSyncSubmit(Lock lock, int executorId, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    abstract void notifyWorkAsync();
+    abstract void awaitExclusive() throws InterruptedException;
+
+    Interruptible.Task task(Mode mode)
+    {
+        return mode == RUN_WITH_LOCK ? this::runWithLock : this::runWithoutLock;
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        if (!lock.tryLock())
+        {
+            submitted.push(async.apply(p1a, p2, p3, p4));
+            notifyWorkAsync();
+            return;
+        }
+
+        try
+        {
+            submitExternalExclusive(sync, async, p1s, p1a, p2, p3, p4);
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.utils.concurrent.LockWithAsyncSignal;
+
+// WARNING: experimental - needs more testing
+class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSubmit
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final LockWithAsyncSignal lock;
+
+    public AccordExecutorAsyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new LockWithAsyncSignal(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorAsyncSubmit(LockWithAsyncSignal lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        lock.clearSignal();
+        if (submitted.isEmpty())
+            lock.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    void notifyWorkAsync()
+    {
+        lock.signal();
+    }
+
+    @Override
+    void notifyWorkExclusive()
+    {
+        lock.signal();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isOwner(Thread.currentThread());
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.IntFunction;
+
+import accord.utils.Invariants;
+import org.agrona.collections.LongHashSet;
+import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.service.accord.AccordExecutor.Mode;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+class AccordExecutorInfiniteLoops implements Shutdownable
+{
+    private final Interruptible[] loops;
+    private final LongHashSet threadIds;
+
+    public AccordExecutorInfiniteLoops(Mode mode, int threads, IntFunction<String> name, Function<Mode, Interruptible.Task> tasks)
+    {
+        Invariants.checkState(mode == RUN_WITH_LOCK ? threads == 1 : threads >= 1);
+        final LongHashSet threadIds = new LongHashSet(threads, 0.5f);
+        this.loops = new Interruptible[threads];
+        for (int i = 0; i < threads; ++i)
+        {
+            loops[i] = executorFactory().infiniteLoop(name.apply(i), tasks.apply(mode), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+            if (loops[i] instanceof InfiniteLoopExecutor)
+                threadIds.add(((InfiniteLoopExecutor) loops[i]).threadId());
+        }
+        this.threadIds = threadIds;
+    }
+
+    public boolean isInLoop()
+    {
+        return threadIds.contains(Thread.currentThread().getId());
+    }
+
+    @Override
+    public void shutdown()
+    {
+        for (Interruptible loop : loops)
+            loop.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        for (Interruptible loop : loops)
+            loop.shutdownNow();
+        return null;
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        for (Interruptible loop : loops)
+        {
+            if (!loop.isTerminated())
+                return false;
+        }
+        return true;
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        long deadline = nanoTime() + unit.toNanos(timeout);
+        for (Interruptible loop : loops)
+        {
+            long wait = deadline - nanoTime();
+            if (!loop.awaitTermination(wait, unit))
+                return false;
+        }
+        return true;
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+// WARNING: experimental - needs more testing
+class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSyncSubmit
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final ReentrantLock lock;
+    private final Condition hasWork;
+
+    public AccordExecutorSemiSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSemiSyncSubmit(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.hasWork = lock.newCondition();
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        if (submitted.isEmpty())
+            hasWork.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    void notifyWorkAsync()
+    {
+        // we check running both sides of tryLock for ordering guarantees
+        boolean hadRunning = isHeldByExecutor;
+        if (lock.tryLock())
+        {
+            try { hasWork.signal(); }
+            finally { lock.unlock(); }
+        }
+        else if (!hadRunning || !isHeldByExecutor)
+        {
+            lock.lock();
+            try { hasWork.signal(); }
+            finally { lock.unlock(); }
+        }
+    }
+    
+    @Override
+    void notifyWorkExclusive()
+    {
+        hasWork.signal();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java

@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import accord.utils.Invariants;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+class AccordExecutorSimple extends AccordExecutor
+{
+    final ExecutorPlus executor;
+    final ReentrantLock lock;
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, Agent agent)
+    {
+        this(executorId, name, metrics, Stage.READ.executor(), Stage.MUTATION.executor(), Stage.READ.executor(), agent);
+    }
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, name, metrics, wrap(loadExecutor), wrap(saveExecutor), wrap(rangeLoadExecutor), agent);
+    }
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, ExecutorFunction loadExecutor, ExecutorFunction saveExecutor, ExecutorFunction rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSimple(ReentrantLock lock, int executorId, String name, AccordCacheMetrics metrics, ExecutorFunction loadExecutor, ExecutorFunction saveExecutor, ExecutorFunction rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, constantFactory(loadExecutor), constantFactory(saveExecutor), constantFactory(rangeLoadExecutor), agent);
+        this.lock = lock;
+        this.executor = executorFactory().sequential(name);
+    }
+
+    public AccordExecutorSimple(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    public AccordExecutorSimple(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        Invariants.checkArgument(threads == 1);
+        this.lock = lock;
+        this.executor = executorFactory().sequential(name.apply(0));
+
+    }
+
+    @Override
+    public boolean hasTasks()
+    {
+        return tasks + executor.getActiveTaskCount() + executor.getPendingTaskCount() > 0;
+    }
+
+    protected void run()
+    {
+        lock.lock();
+        try
+        {
+            running = 1;
+            while (true)
+            {
+                Task task = pollWaitingToRunExclusive();
+                if (task == null)
+                    return;
+
+                --tasks;
+                try { task.preRunExclusive(); task.run(); }
+                catch (Throwable t) { task.fail(t); }
+                finally { task.cleanupExclusive(); }
+            }
+        }
+        catch (Throwable t)
+        {
+            throw t;
+        }
+        finally
+        {
+            running = 0;
+            if (hasWaitingToRun())
+                executor.execute(this::run);
+            lock.unlock();
+        }
+    }
+
+    @Override
+    <P1s, P1a, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        lock.lock();
+        try
+        {
+            sync.accept(this, p1s, p2, p3, p4);
+        }
+        finally
+        {
+            if (hasWaitingToRun())
+                executor.execute(this::run);
+
+            lock.unlock();
+        }
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return executor.isTerminated();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        executor.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return executor.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    {
+        return executor.awaitTermination(timeout, units);
+    }
+
+}

src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final ReentrantLock lock;
+    private final Condition hasWork;
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, String name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, mode, 1, constant(name), metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, mode, threads, name, metrics, constantFactory(loadExecutor), constantFactory(saveExecutor), constantFactory(rangeLoadExecutor), agent);
+    }
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSyncSubmit(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.hasWork = lock.newCondition();
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        hasWork.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    void notifyWorkExclusive()
+    {
+        hasWork.signal();
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        lock.lock();
+        try
+        {
+            submitExternalExclusive(sync, async, p1s, p1a, p2, p3, p4);
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java

@@ -203,7 +203,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
             future.addCallback((state, fail) -> {
                 if (fail == null) success(from, Ranges.of(range));
                 else fail(from, Ranges.of(range), fail);
-            }, ((AccordCommandStore) commandStore()).executor());
+            }, ((AccordCommandStore) commandStore()).taskExecutor());
         }
     }
 

src/java/org/apache/cassandra/service/accord/AccordJournal.java

@@ -62,6 +62,7 @@ import org.apache.cassandra.journal.ValueSerializer;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.accord.AccordJournalValueSerializers.IdentityAccumulator;
 import org.apache.cassandra.service.accord.JournalKey.JournalKeySupport;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.utils.ExecutorUtils;
 
 import static accord.primitives.SaveStatus.ErasedOrVestigial;
@@ -86,14 +87,16 @@ public class AccordJournal implements IJournal, Shutdownable
     private final Journal<JournalKey, Object> journal;
     private final AccordJournalTable<JournalKey, Object> journalTable;
     private final Params params;
+    private final AccordAgent agent;
     Node node;
 
     enum Status { INITIALIZED, STARTING, REPLAY, STARTED, TERMINATING, TERMINATED }
     private volatile Status status = Status.INITIALIZED;
 
     @VisibleForTesting
-    public AccordJournal(Params params)
+    public AccordJournal(Params params, AccordAgent agent)
     {
+        this.agent = agent;
         File directory = new File(DatabaseDescriptor.getAccordJournalDirectory());
         this.journal = new Journal<>("AccordJournal", directory, params, JournalKey.SUPPORT,
                                      // In Accord, we are using streaming serialization, i.e. Reader/Writer interfaces instead of materializing objects
@@ -180,7 +183,7 @@ public class AccordJournal implements IJournal, Shutdownable
     public Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
     {
         SavedCommand.Builder builder = loadDiffs(commandStoreId, txnId);
-        Cleanup cleanup = builder.shouldCleanup(redundantBefore, durableBefore);
+        Cleanup cleanup = builder.shouldCleanup(agent, redundantBefore, durableBefore);
         switch (cleanup)
         {
             case EXPUNGE_PARTIAL:
@@ -195,7 +198,10 @@ public class AccordJournal implements IJournal, Shutdownable
     public SavedCommand.MinimalCommand loadMinimal(int commandStoreId, TxnId txnId, SavedCommand.Load load, RedundantBefore redundantBefore, DurableBefore durableBefore)
     {
         SavedCommand.Builder builder = loadDiffs(commandStoreId, txnId, load);
-        Cleanup cleanup = builder.shouldCleanup(redundantBefore, durableBefore);
+        if (!builder.nextCalled)
+            return null;
+
+        Cleanup cleanup = builder.shouldCleanup(node.agent(), redundantBefore, durableBefore);
         switch (cleanup)
         {
             case EXPUNGE_PARTIAL:
@@ -203,6 +209,7 @@ public class AccordJournal implements IJournal, Shutdownable
             case ERASE:
                 return null;
         }
+        Invariants.checkState(builder.saveStatus != null, "No saveSatus loaded, but next was called and cleanup was not: %s", builder);
         return builder.asMinimal();
     }
 
@@ -235,7 +242,7 @@ public class AccordJournal implements IJournal, Shutdownable
     }
 
     @Override
-    public void appendCommand(int store, SavedCommand.DiffWriter value, Runnable onFlush)
+    public void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush)
     {
         if (value == null || status == Status.REPLAY)
         {
@@ -248,7 +255,7 @@ public class AccordJournal implements IJournal, Shutdownable
         JournalKey key = new JournalKey(value.key(), JournalKey.Type.COMMAND_DIFF, store);
         RecordPointer pointer = journal.asyncWrite(key, value, SENTINEL_HOSTS);
         if (onFlush != null)
-            journal.onFlush(pointer, onFlush);
+            journal.onDurable(pointer, onFlush);
     }
 
     @Override
@@ -266,7 +273,7 @@ public class AccordJournal implements IJournal, Shutdownable
                 JournalKey key = new JournalKey(TxnId.NONE, JournalKey.Type.DURABLE_BEFORE, 0);
                 RecordPointer pointer = appendInternal(key, addDurableBefore);
                 // TODO (required): what happens on failure?
-                journal.onFlush(pointer, () -> result.setSuccess(null));
+                journal.onDurable(pointer, () -> result.setSuccess(null));
                 return result;
             }
 
@@ -297,7 +304,7 @@ public class AccordJournal implements IJournal, Shutdownable
             return;
 
         if (pointer != null)
-            journal.onFlush(pointer, onFlush);
+            journal.onDurable(pointer, onFlush);
         else
             onFlush.run();
     }
@@ -459,7 +466,7 @@ public class AccordJournal implements IJournal, Shutdownable
                         }
                     });
 
-                    Cleanup cleanup = builder.shouldCleanup(compactionInfo.redundantBefores.get(key.commandStoreId), compactionInfo.durableBefores.get(key.commandStoreId));
+                    Cleanup cleanup = builder.shouldCleanup(node.agent(), compactionInfo.redundantBefores.get(key.commandStoreId), compactionInfo.durableBefores.get(key.commandStoreId));
                     switch (cleanup)
                     {
                         case ERASE:

src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java

@@ -55,7 +55,7 @@ public class AccordJournalValueSerializers
     }
 
     public static class CommandDiffSerializer
-    implements FlyweightSerializer<SavedCommand.DiffWriter, SavedCommand.Builder>
+    implements FlyweightSerializer<SavedCommand.Writer, SavedCommand.Builder>
     {
         @Override
         public SavedCommand.Builder mergerFor(JournalKey journalKey)
@@ -64,7 +64,7 @@ public class AccordJournalValueSerializers
         }
 
         @Override
-        public void serialize(JournalKey key, SavedCommand.DiffWriter writer, DataOutputPlus out, int userVersion)
+        public void serialize(JournalKey key, SavedCommand.Writer writer, DataOutputPlus out, int userVersion)
         {
             try
             {

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -29,6 +29,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -57,8 +58,6 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.topology.Topology;
 import accord.utils.Invariants;
-import accord.utils.async.Observable;
-import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
@@ -111,12 +110,14 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.accord.RouteIndex;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.LocalVersionedSerializer;
 import org.apache.cassandra.io.MessageVersionProvider;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
 import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -149,6 +150,7 @@ import org.apache.cassandra.utils.concurrent.OpOrder;
 import static accord.utils.Invariants.checkArgument;
 import static accord.utils.Invariants.checkState;
 import static java.lang.String.format;
+import static java.util.Collections.emptyMap;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate;
 import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow;
@@ -183,7 +185,6 @@ public class AccordKeyspace
 
     private static final ClusteringIndexFilter FULL_PARTITION = new ClusteringIndexNamesFilter(BTreeSet.of(new ClusteringComparator(), Clustering.EMPTY), false);
 
-    //TODO (now, performance): should this be partitioner rather than TableId?  As of this patch distributed tables should only have 1 partitioner...
     private static final ConcurrentMap<TableId, AccordRoutingKeyByteSource.Serializer> TABLE_SERIALIZERS = new ConcurrentHashMap<>();
 
     private static AccordRoutingKeyByteSource.Serializer getRoutingKeySerializer(AccordRoutingKey key)
@@ -239,7 +240,10 @@ public class AccordKeyspace
               + "user_version int,"
               + "record blob,"
               + "PRIMARY KEY((store_id, type, id), descriptor, offset)"
-              + ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC) WITH compression = {'class':'NoopCompressor'};")
+              + ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC)" +
+              " WITH compression = {'class':'NoopCompressor'};")
+        .compaction(CompactionParams.lcs(emptyMap()))
+        .bloomFilterFpChance(0.01)
         .partitioner(new LocalPartitioner(BytesType.instance))
         .build();
 
@@ -478,7 +482,7 @@ public class AccordKeyspace
               + format("last_write_timestamp %s, ", TIMESTAMP_TUPLE)
               + "PRIMARY KEY((store_id, routing_key))"
               + ')')
-        .partitioner(new LocalPartitioner(CompositeType.getInstance(Int32Type.instance, BytesType.instance)))
+        .partitioner(new LocalCompositePrefixPartitioner(Int32Type.instance, BytesType.instance))
         .build();
 
     public static class TimestampsForKeyColumns
@@ -490,8 +494,14 @@ public class AccordKeyspace
         public static final ColumnMetadata last_executed_timestamp = getColumn(TimestampsForKeys, "last_executed_timestamp");
         public static final ColumnMetadata last_executed_micros = getColumn(TimestampsForKeys, "last_executed_micros");
         public static final ColumnMetadata last_write_timestamp = getColumn(TimestampsForKeys, "last_write_timestamp");
-
+        public static final ColumnMetadata last_write_id = getColumn(TimestampsForKeys, "last_write_id");
         static final Columns columns = Columns.from(Lists.newArrayList(last_executed_timestamp, last_executed_micros, last_write_timestamp));
+        static final ColumnFilter allColumns = ColumnFilter.all(TimestampsForKeys);
+
+        static DecoratedKey decorateKey(int storeId, RoutingKey key)
+        {
+            return TimestampsForKeys.partitioner.decorateKey(makeKey(storeId, key));
+        }
 
         static ByteBuffer makeKey(int storeId, RoutingKey key)
         {
@@ -517,6 +527,11 @@ public class AccordKeyspace
             return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
         }
 
+        public static TokenKey getKey(DecoratedKey key)
+        {
+            return getKey(splitPartitionKey(key));
+        }
+
         public static TokenKey getKey(ByteBuffer[] partitionKeyComponents)
         {
             return (TokenKey) deserializeRoutingKey(partitionKeyComponents[routing_key.position()]);
@@ -580,7 +595,7 @@ public class AccordKeyspace
         }
     }
 
-    private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance, BytesType.instance);
+    private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance);
     private static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
 
     private static TableMetadata commandsForKeysTable(String tableName)
@@ -596,6 +611,8 @@ public class AccordKeyspace
               + ')'
                + " WITH compression = {'class':'NoopCompressor'};")
         .partitioner(CFKPartitioner)
+        .compaction(CompactionParams.lcs(emptyMap()))
+        .bloomFilterFpChance(0.01)
         .build();
     }
 
@@ -983,7 +1000,7 @@ public class AccordKeyspace
     public static void findAllKeysBetween(int commandStore,
                                           AccordRoutingKey start, boolean startInclusive,
                                           AccordRoutingKey end, boolean endInclusive,
-                                          Observable<TokenKey> callback)
+                                          Consumer<TokenKey> consumer)
     {
 
         Token startToken = CommandsForKeysAccessor.getPrefixToken(commandStore, start);
@@ -1006,41 +1023,23 @@ public class AccordKeyspace
         else
             bounds = new ExcludingBounds<>(startPosition, endPosition);
 
-        Stage.READ.executor().submit(() -> {
-            ColumnFamilyStore baseCfs = Keyspace.openAndGetStore(CommandsForKeys);
-            try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
-                 WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
-                 CloseableIterator<DecoratedKey> iter = LocalCompositePrefixPartitioner.keyIterator(CommandsForKeys, bounds))
+        ColumnFamilyStore baseCfs = AccordColumnFamilyStores.commandsForKey;
+        try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
+             WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
+             CloseableIterator<DecoratedKey> iter = LocalCompositePrefixPartitioner.keyIterator(CommandsForKeys, bounds))
+        {
+            // Need the second try to handle callback errors vs read errors.
+            // Callback will see the read errors, but if the callback fails the outer try will see those errors
+            while (iter.hasNext())
             {
-                // Need the second try to handle callback errors vs read errors.
-                // Callback will see the read errors, but if the callback fails the outer try will see those errors
-                try
-                {
-                    while (iter.hasNext())
-                    {
-                        TokenKey pk = CommandsForKeysAccessor.getKey(iter.next());
-                        callback.onNext(pk);
-                    }
-                    callback.onCompleted();
-                }
-                catch (Exception e)
-                {
-                    callback.onError(e);
-                }
+                TokenKey pk = CommandsForKeysAccessor.getKey(iter.next());
+                consumer.accept(pk);
             }
-            catch (IOException e)
-            {
-                try
-                {
-                    callback.onError(e);
-                }
-                catch (Throwable t)
-                {
-                    e.addSuppressed(t);
-                }
-                throw new RuntimeException(e);
-            }
-        });
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     public static TxnId deserializeTxnId(UntypedResultSet.Row row)
@@ -1097,7 +1096,7 @@ public class AccordKeyspace
         return (TokenKey) AccordRoutingKeyByteSource.Serializer.fromComparableBytes(ByteBufferAccessor.instance, tokenBytes, tableId, currentVersion, null);
     }
 
-    public static Mutation getTimestampsForKeyMutation(int storeId, TimestampsForKey current, long timestampMicros)
+    public static PartitionUpdate getTimestampsForKeyUpdate(int storeId, TimestampsForKey current, long timestampMicros)
     {
         try
         {
@@ -1118,8 +1117,7 @@ public class AccordKeyspace
                 return null;
 
             ByteBuffer key = TimestampsForKeyColumns.makeKey(storeId, current.key());
-            PartitionUpdate update = singleRowUpdate(TimestampsForKeys, key, row);
-            return new Mutation(update);
+            return singleRowUpdate(TimestampsForKeys, key, row);
         }
         catch (IOException e)
         {
@@ -1127,50 +1125,79 @@ public class AccordKeyspace
         }
     }
 
-    public static Mutation getTimestampsForKeyMutation(AccordCommandStore commandStore, AccordSafeTimestampsForKey liveTimestamps, long timestampMicros)
+    public static Runnable getTimestampsForKeyUpdater(AccordCommandStore commandStore, TimestampsForKey liveTimestamps, long timestampMicros)
     {
-        return getTimestampsForKeyMutation(commandStore.id(), liveTimestamps.current(), timestampMicros);
+        PartitionUpdate upd = getTimestampsForKeyUpdate(commandStore.id(), liveTimestamps, timestampMicros);
+        return () -> {
+            ColumnFamilyStore cfs = AccordColumnFamilyStores.timestampsForKey;
+            try (OpOrder.Group group = Keyspace.writeOrder.start())
+            {
+                cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
+            }
+        };
     }
 
-    public static UntypedResultSet loadTimestampsForKeyRow(CommandStore commandStore, TokenKey key)
+    public static UntypedResultSet loadTimestampsForKeyRow(int commandStoreId, TokenKey key)
     {
         String cql = "SELECT * FROM " + ACCORD_KEYSPACE_NAME + '.' + TIMESTAMPS_FOR_KEY + ' ' +
                      "WHERE store_id = ? " +
                      "AND routing_key = ?";
 
-        return executeInternal(cql, commandStore.id(), serializeRoutingKey(key));
+        return executeInternal(cql, commandStoreId, serializeRoutingKey(key));
     }
 
-    public static TimestampsForKey loadTimestampsForKey(AccordCommandStore commandStore, TokenKey key)
+    private static SinglePartitionReadCommand getTimestampsForKeyRead(int storeId, TokenKey key, long nowInSeconds)
     {
-        commandStore.checkNotInStoreThread();
-        return unsafeLoadTimestampsForKey(commandStore, key);
+        return SinglePartitionReadCommand.create(TimestampsForKeys, nowInSeconds,
+                                                 TimestampsForKeyColumns.allColumns,
+                                                 RowFilter.none(),
+                                                 DataLimits.NONE,
+                                                 TimestampsForKeyColumns.decorateKey(storeId, key),
+                                                 FULL_PARTITION);
     }
 
-    public static TimestampsForKey unsafeLoadTimestampsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static TimestampsForKey loadTimestampsForKey(int commandStoreId, TokenKey key)
     {
-        UntypedResultSet rows = loadTimestampsForKeyRow(commandStore, key);
+        return unsafeLoadTimestampsForKey(commandStoreId, key);
+    }
 
-        if (rows.isEmpty())
+    public static TimestampsForKey unsafeLoadTimestampsForKey(int commandStoreId, TokenKey key)
+    {
+        long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
+        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
+
+        SinglePartitionReadCommand command = getTimestampsForKeyRead(commandStoreId, key, nowInSeconds);
+        try (ReadExecutionController controller = command.executionController();
+             FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
         {
-            return null;
+            if (!partitions.hasNext())
+                return null;
+
+            try (RowIterator partition = partitions.next())
+            {
+                Invariants.checkState(partition.hasNext());
+                Row row = partition.next();
+                TokenKey checkKey = TimestampsForKeyRows.getKey(partition.partitionKey());
+                checkState(checkKey.equals(key));
+
+                Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_executed_timestamp), ByteBufferAccessor.instance, Timestamp::fromBits, Timestamp.NONE);
+                ByteBuffer lastExecutedMicrosBB = cellValue(row, TimestampsForKeyColumns.last_executed_micros);
+                long lastExecutedMicros = lastExecutedMicrosBB == null || !lastExecutedMicrosBB.hasRemaining() ? 0 : lastExecutedMicrosBB.getLong(lastExecutedMicrosBB.position());
+                TxnId lastWriteId = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_write_id), ByteBufferAccessor.instance, TxnId::fromBits, TxnId.NONE);
+                Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_write_timestamp), ByteBufferAccessor.instance, Timestamp::fromBits, Timestamp.NONE);
+                return TimestampsForKey.SerializerSupport.create(key, lastExecutedTimestamp, lastExecutedMicros, lastWriteId, lastWriteTimestamp);
+            }
+        }
+        catch (Throwable t)
+        {
+            logger.error("Exception loading AccordTimestampsForKey " + key, t);
+            throw t;
         }
-
-        UntypedResultSet.Row row = rows.one();
-        TokenKey checkKey = (TokenKey) deserializeRoutingKey(row.getBytes("routing_key"));
-        checkState(checkKey.equals(key));
-
-        Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(row, "last_executed_timestamp", Timestamp::fromBits, Timestamp.NONE);
-        long lastExecutedMicros = row.has("last_executed_micros") ? row.getLong("last_executed_micros") : 0;
-        TxnId lastWriteId = deserializeTimestampOrDefault(row, "last_write_id", TxnId::fromBits, TxnId.NONE);
-        Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(row, "last_write_timestamp", Timestamp::fromBits, Timestamp.NONE);
-
-        return TimestampsForKey.SerializerSupport.create(key, lastExecutedTimestamp, lastExecutedMicros, lastWriteId, lastWriteTimestamp);
     }
 
-    private static DecoratedKey makeKeySeparateTable(CommandsForKeyAccessor accessor, int storeId, TokenKey key)
+    private static DecoratedKey makeKeySeparateTable(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
     {
-        ByteBuffer pk = accessor.keyComparator.make(storeId,
+        ByteBuffer pk = accessor.keyComparator.make(commandStoreId,
                                                     UUIDSerializer.instance.serialize(key.table().asUUID()),
                                                     serializeRoutingKeyNoTable(key)).serializeAsPartitionKey();
         return accessor.table.partitioner.decorateKey(pk);
@@ -1209,9 +1236,11 @@ public class AccordKeyspace
         return SchemaHolder.schema.getTablePartitioner(tableId);
     }
 
-    private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, long timestampMicros)
+    private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, Object serialized, long timestampMicros)
     {
-        ByteBuffer bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
+        ByteBuffer bytes;
+        if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized;
+        else bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
         return getCommandsForKeyPartitionUpdate(storeId, key, timestampMicros, bytes);
     }
 
@@ -1223,9 +1252,16 @@ public class AccordKeyspace
                                singleCellRow(Clustering.EMPTY, BufferCell.live(CommandsForKeysAccessor.data, timestampMicros, bytes)));
     }
 
-    public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKey update, long timestampMicros)
+    public static Runnable getCommandsForKeyUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros)
     {
-        return new Mutation(getCommandsForKeyPartitionUpdate(storeId, (TokenKey)update.key(), update, timestampMicros));
+        PartitionUpdate upd = getCommandsForKeyPartitionUpdate(storeId, key, update, serialized, timestampMicros);
+        return () -> {
+            ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey;
+            try (OpOrder.Group group = Keyspace.writeOrder.start())
+            {
+                cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
+            }
+        };
     }
 
     private static <T> ByteBuffer cellValue(Cell<T> cell)
@@ -1260,12 +1296,12 @@ public class AccordKeyspace
         return getCommandsForKeyRead(CommandsForKeysAccessor, storeId, key, nowInSeconds);
     }
 
-    static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, AccordCommandStore commandStore, TokenKey key)
+    static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
     {
         long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
         int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
 
-        SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStore.id(), key, nowInSeconds);
+        SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStoreId, key, nowInSeconds);
 
         try (ReadExecutionController controller = command.executionController();
              FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
@@ -1288,15 +1324,14 @@ public class AccordKeyspace
         }
     }
 
-    public static CommandsForKey unsafeLoadCommandsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static CommandsForKey unsafeLoadCommandsForKey(int commandStoreId, TokenKey key)
     {
-        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStoreId, key);
     }
 
-    public static CommandsForKey loadCommandsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static CommandsForKey loadCommandsForKey(int commandStoreId, TokenKey key)
     {
-        commandStore.checkNotInStoreThread();
-        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStoreId, key);
     }
 
     public static class EpochDiskState
@@ -1626,4 +1661,11 @@ public class AccordKeyspace
         TABLE_SERIALIZERS.clear();
         SchemaHolder.schema = Schema.instance;
     }
+
+    public static class AccordColumnFamilyStores
+    {
+        public static final ColumnFamilyStore journal = Schema.instance.getColumnFamilyStoreInstance(Journal.id);
+        public static final ColumnFamilyStore commandsForKey = Schema.instance.getColumnFamilyStoreInstance(CommandsForKeys.id);
+        public static final ColumnFamilyStore timestampsForKey = Schema.instance.getColumnFamilyStoreInstance(TimestampsForKeys.id);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordMessageSink.java

@@ -272,7 +272,7 @@ public class AccordMessageSink implements MessageSink
         long delayedAtNanos = Long.MAX_VALUE;
         long expiresAtNanos;
         if (isRangeBarrier(request))
-            expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos();
+            expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos();
         else
             expiresAtNanos = nowNanos + verb.expiresAfterNanos();
 

src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java

@@ -379,6 +379,7 @@ public class AccordObjectSizes
         for (int i = 0 ; i < cfk.size() ; ++i)
         {
             TxnInfo info = cfk.get(i);
+            if (info.executeAt != info) size += TIMESTAMP_SIZE;
             if (info.getClass() != TxnInfoExtra.class) continue;
             TxnInfoExtra infoExtra = (TxnInfoExtra) info;
             if (infoExtra.missing.length > 0)

src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java

@@ -19,25 +19,20 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.Objects;
-import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import accord.local.Command;
 import accord.local.SafeCommand;
 import accord.primitives.TxnId;
-import accord.utils.Invariants;
 import org.apache.cassandra.utils.concurrent.Ref;
 
-import static accord.utils.Invariants.Paranoia.LINEAR;
-import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
-
 public class AccordSafeCommand extends SafeCommand implements AccordSafeState<TxnId, Command>
 {
     public static class DebugAccordSafeCommand extends AccordSafeCommand
     {
         final Ref<?> selfRef;
-        public DebugAccordSafeCommand(AccordCachingState<TxnId, Command> global)
+        public DebugAccordSafeCommand(AccordCacheEntry<TxnId, Command> global)
         {
             super(global);
             selfRef = new Ref<>(this, null);
@@ -58,11 +53,11 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     }
 
     private boolean invalidated;
-    private final AccordCachingState<TxnId, Command> global;
+    private final AccordCacheEntry<TxnId, Command> global;
     private Command original;
     private Command current;
 
-    public AccordSafeCommand(AccordCachingState<TxnId, Command> global)
+    public AccordSafeCommand(AccordCacheEntry<TxnId, Command> global)
     {
         super(global.key());
         this.global = global;
@@ -97,7 +92,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     }
 
     @Override
-    public AccordCachingState<TxnId, Command> global()
+    public AccordCacheEntry<TxnId, Command> global()
     {
         checkNotInvalidated();
         return global;
@@ -125,7 +120,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
         return original;
     }
 
-    public SavedCommand.DiffWriter diff()
+    public SavedCommand.Writer diff()
     {
         return SavedCommand.diff(original, current);
     }
@@ -134,8 +129,10 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            uninitialised();
     }
 
     @Override
@@ -149,9 +146,4 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     {
         return invalidated;
     }
-
-    public static Function<AccordCachingState<TxnId, Command>, AccordSafeCommand> safeRefFactory()
-    {
-        return Invariants.testParanoia(LINEAR, LINEAR, HIGH) ? DebugAccordSafeCommand::new : AccordSafeCommand::new;
-    }
 }

src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java

@@ -18,6 +18,9 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
@@ -31,45 +34,48 @@ import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.api.RoutingKey;
-import accord.impl.AbstractSafeCommandStore;
 import accord.impl.CommandsSummary;
+import accord.impl.SafeTimestampsForKey;
 import accord.local.CommandStores;
 import accord.local.CommandStores.RangesForEpoch;
+import accord.local.KeyHistory;
 import accord.local.NodeCommandStoreService;
 import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
+import accord.local.SafeCommandStore;
 import accord.local.cfk.CommandsForKey;
 import accord.primitives.AbstractKeys;
 import accord.primitives.AbstractRanges;
 import accord.primitives.Ranges;
+import accord.primitives.Routable;
 import accord.primitives.Routables;
+import accord.primitives.RoutingKeys;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.utils.Invariants;
+import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
 
-public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey>
+import static accord.local.KeyHistory.TIMESTAMPS;
+import static accord.utils.Invariants.illegalArgument;
+import static accord.utils.Invariants.illegalState;
+
+public class AccordSafeCommandStore extends SafeCommandStore
 {
-    private final Map<TxnId, AccordSafeCommand> commands;
-    private final Map<RoutingKey, AccordSafeCommandsForKey> commandsForKeys;
-    private final Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKeys;
-    private final @Nullable AccordSafeCommandsForRanges commandsForRanges;
+    final AccordTask<?> task;
+    final PreLoadContext context;
+    private final @Nullable CommandsForRanges commandsForRanges;
     private final AccordCommandStore commandStore;
     private RangesForEpoch ranges;
     private FieldUpdates fieldUpdates;
 
-    private AccordSafeCommandStore(PreLoadContext context,
-                                   Map<TxnId, AccordSafeCommand> commands,
-                                   Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey,
-                                   Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey,
-                                   @Nullable AccordSafeCommandsForRanges commandsForRanges,
+    private AccordSafeCommandStore(AccordTask<?> task,
+                                   @Nullable CommandsForRanges commandsForRanges,
                                    AccordCommandStore commandStore)
     {
-        super(context);
-        this.commands = commands;
-        this.timestampsForKeys = timestampsForKey;
-        this.commandsForKeys = commandsForKey;
+        this.context = task.preLoadContext();
+        this.task = task;
         this.commandsForRanges = commandsForRanges;
         this.commandStore = commandStore;
         commandStore.updateRangesForEpoch(this);
@@ -77,80 +83,223 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
             this.ranges = Invariants.nonNull(commandStore.unsafeRangesForEpoch());
     }
 
-    public static AccordSafeCommandStore create(PreLoadContext preLoadContext,
-                                                Map<TxnId, AccordSafeCommand> commands,
-                                                Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey,
-                                                Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey,
-                                                @Nullable AccordSafeCommandsForRanges commandsForRanges,
+    public static AccordSafeCommandStore create(AccordTask<?> operation,
+                                                @Nullable CommandsForRanges commandsForRanges,
                                                 AccordCommandStore commandStore)
     {
-        return new AccordSafeCommandStore(preLoadContext, commands, timestampsForKey, commandsForKey, commandsForRanges, commandStore);
+        return new AccordSafeCommandStore(operation, commandsForRanges, commandStore);
+    }
+
+    @Override
+    public PreLoadContext canExecute(PreLoadContext context)
+    {
+        if (context.isEmpty()) return context;
+        if (context.keys().domain() == Routable.Domain.Range)
+            return context.isSubsetOf(this.context) ? context : null;
+
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return context.isSubsetOf(this.context) ? context : null;
+
+            for (TxnId txnId : context.txnIds())
+            {
+                if (null != getInternal(txnId))
+                    continue;
+
+                AccordSafeCommand safeCommand = caches.commands().acquireIfLoaded(txnId);
+                if (safeCommand == null)
+                    return null;
+
+                add(safeCommand, caches);
+            }
+
+            KeyHistory keyHistory = context.keyHistory();
+            if (keyHistory == KeyHistory.NONE)
+                return context;
+
+            List<RoutingKey> unavailable = null;
+            Unseekables<?> keys = context.keys();
+            if (keys.size() == 0)
+                return context;
+
+            for (int i = 0 ; i < keys.size() ; ++i)
+            {
+                RoutingKey key = (RoutingKey) keys.get(i);
+                if (keyHistory == TIMESTAMPS)
+                {
+                    if (null != timestampsForKeyInternal(key))
+                        continue; // already in working set
+
+                    AccordSafeTimestampsForKey safeTfk = caches.timestampsForKeys().acquireIfLoaded(key);
+                    if (safeTfk != null)
+                    {
+                        add(safeTfk, caches);
+                        continue;
+                    }
+                }
+                else
+                {
+                    if (null != getInternal(key))
+                        continue; // already in working set
+
+                    AccordSafeCommandsForKey safeCfk = caches.commandsForKeys().acquireIfLoaded(key);
+                    if (safeCfk != null)
+                    {
+                        add(safeCfk, caches);
+                        continue;
+                    }
+                }
+                if (unavailable == null)
+                    unavailable = new ArrayList<>();
+                unavailable.add(key);
+            }
+
+            if (unavailable == null)
+                return context;
+
+            if (unavailable.size() == keys.size())
+                return null;
+
+            return PreLoadContext.contextFor(context.primaryTxnId(), context.additionalTxnId(), keys.without(RoutingKeys.ofSortedUnique(unavailable)), keyHistory);
+        }
+    }
+
+    @Override
+    public PreLoadContext context()
+    {
+        return context;
     }
 
     @VisibleForTesting
     public Set<RoutingKey> commandsForKeysKeys()
     {
-        return commandsForKeys.keySet();
+        if (task.commandsForKey() == null)
+            return Collections.emptySet();
+        return task.commandsForKey().keySet();
     }
 
     @Override
-    protected AccordSafeCommand getCommandInternal(TxnId txnId)
+    protected AccordSafeCommand getInternal(TxnId txnId)
     {
+        Map<TxnId, AccordSafeCommand> commands = task.commands();
+        if (commands == null)
+            return null;
         return commands.get(txnId);
     }
 
     @Override
-    protected void addCommandInternal(AccordSafeCommand command)
+    protected AccordSafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId)
     {
-        commands.put(command.txnId(), command);
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return null;
+
+            AccordSafeCommand command = caches.commands().acquireIfLoaded(txnId);
+            if (command == null)
+                return null;
+
+            return add(command, caches);
+        }
+    }
+
+    private AccordSafeCommand add(AccordSafeCommand safeCommand, ExclusiveCaches caches)
+    {
+        Object check = task.ensureCommands().putIfAbsent(safeCommand.txnId(), safeCommand);
+        if (check == null)
+        {
+            safeCommand.preExecute();
+            return safeCommand;
+        }
+        else
+        {
+            caches.commands().release(safeCommand, task);
+            throw illegalState("Attempted to take a duplicate reference to %s", safeCommand.txnId());
+        }
+    }
+
+    private AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, ExclusiveCaches caches)
+    {
+        Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk);
+        if (check == null)
+        {
+            safeCfk.preExecute();
+            return safeCfk;
+        }
+        else
+        {
+            caches.commandsForKeys().release(safeCfk, task);
+            throw illegalState("Attempted to take a duplicate reference to CFK for %s", safeCfk.key());
+        }
+    }
+
+    private AccordSafeTimestampsForKey add(AccordSafeTimestampsForKey safeTfk, ExclusiveCaches caches)
+    {
+        Object check = task.ensureTimestampsForKey().putIfAbsent(safeTfk.key(), safeTfk);
+        if (check == null)
+        {
+            safeTfk.preExecute();
+            return safeTfk;
+        }
+        else
+        {
+            caches.timestampsForKeys().release(safeTfk, task);
+            throw illegalState("Attempted to take a duplicate reference to CFK for %s", safeTfk.key());
+        }
     }
 
     @Override
-    protected AccordSafeCommand getIfLoaded(TxnId txnId)
+    protected AccordSafeCommandsForKey getInternal(RoutingKey key)
     {
-        AccordSafeCommand command = commandStore.commandCache().acquireIfLoaded(txnId);
-        if (command != null) command.preExecute();
-        return command;
+        Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey = task.commandsForKey();
+        if (commandsForKey == null)
+            return null;
+        return commandsForKey.get(key);
     }
 
     @Override
-    protected AccordSafeCommandsForKey getCommandsForKeyInternal(RoutingKey key)
+    protected AccordSafeCommandsForKey ifLoadedInternal(RoutingKey key)
     {
-        return commandsForKeys.get(key);
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return null;
+
+            AccordSafeCommandsForKey safeCfk = caches.commandsForKeys().acquireIfLoaded(key);
+            if (safeCfk == null)
+                return null;
+
+            Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk);
+            if (check == null)
+            {
+                safeCfk.preExecute();
+                return safeCfk;
+            }
+            else
+            {
+                caches.commandsForKeys().release(safeCfk, task);
+                throw illegalState("Attempted to take a duplicate reference to CFK for %s", key);
+            }
+        }
     }
 
     @Override
-    protected void addCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
+    public SafeTimestampsForKey timestampsForKey(RoutingKey key)
     {
-        commandsForKeys.put(cfk.key(), cfk);
+        AccordSafeTimestampsForKey safeTfk = timestampsForKeyInternal(key);
+        if (safeTfk == null)
+            throw illegalArgument("%s not referenced in %s", key, context);
+        return safeTfk;
     }
 
-    @Override
-    protected AccordSafeCommandsForKey getCommandsForKeyIfLoaded(RoutingKey key)
+    private AccordSafeTimestampsForKey timestampsForKeyInternal(RoutingKey key)
     {
-        AccordSafeCommandsForKey cfk = commandStore.commandsForKeyCache().acquireIfLoaded(key);
-        if (cfk != null) cfk.preExecute();
-        return cfk;
-    }
+        Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey = task.timestampsForKey();
+        if (timestampsForKey == null)
+            return null;
 
-    @Override
-    protected AccordSafeTimestampsForKey getTimestampsForKeyInternal(RoutingKey key)
-    {
-        return timestampsForKeys.get(key);
-    }
-
-    @Override
-    protected void addTimestampsForKeyInternal(AccordSafeTimestampsForKey cfk)
-    {
-        timestampsForKeys.put(cfk.key(), cfk);
-    }
-
-    @Override
-    protected AccordSafeTimestampsForKey getTimestampsForKeyIfLoaded(RoutingKey key)
-    {
-        AccordSafeTimestampsForKey cfk = commandStore.timestampsForKeyCache().acquireIfLoaded(key);
-        if (cfk != null) cfk.preExecute();
-        return cfk;
+        return timestampsForKey.get(key);
     }
 
     @Override
@@ -200,27 +349,27 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
     {
         if (commandsForRanges == null)
             return accumulate;
-        CommandsForRanges cfr = commandsForRanges.current();
+
         switch (keysOrRanges.domain())
         {
             case Key:
             {
                 AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
-                if (!cfr.ranges.intersects(keys))
+                if (!commandsForRanges.ranges.intersects(keys))
                     return accumulate;
             }
             break;
             case Range:
             {
                 AbstractRanges ranges = (AbstractRanges) keysOrRanges;
-                if (!cfr.ranges.intersects(ranges))
+                if (!commandsForRanges.ranges.intersects(ranges))
                     return accumulate;
             }
             break;
             default:
                 throw new AssertionError("Unknown domain: " + keysOrRanges.domain());
         }
-        return map.apply(cfr, accumulate);
+        return map.apply(commandsForRanges, accumulate);
     }
 
     private <O> O mapReduceForKey(Routables<?> keysOrRanges, BiFunction<CommandsSummary, O, O> map, O accumulate)
@@ -246,7 +395,12 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
                 // are contained within the ranges... so walk all keys found in commandsForKeys
                 if (!context.keys().containsAll(keysOrRanges))
                     throw new AssertionError("Range(s) detected not present in the PreLoadContext: expected " + context.keys() + " but given " + keysOrRanges);
-                for (RoutingKey key : commandsForKeys.keySet())
+
+                Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey = task.commandsForKey();
+                if (commandsForKey == null)
+                    break;
+
+                for (RoutingKey key : commandsForKey.keySet())
                 {
                     //TODO (duplicate code): this is a repeat of Key... only change is checking contains in range
                     CommandsForKey commands = get(key).current();

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java

@@ -29,11 +29,11 @@ import accord.local.cfk.SafeCommandsForKey;
 public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState<RoutingKey, CommandsForKey>
 {
     private boolean invalidated;
-    private final AccordCachingState<RoutingKey, CommandsForKey> global;
+    private final AccordCacheEntry<RoutingKey, CommandsForKey> global;
     private CommandsForKey original;
     private CommandsForKey current;
 
-    public AccordSafeCommandsForKey(AccordCachingState<RoutingKey, CommandsForKey> global)
+    public AccordSafeCommandsForKey(AccordCacheEntry<RoutingKey, CommandsForKey> global)
     {
         super(global.key());
         this.global = global;
@@ -82,7 +82,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     }
 
     @Override
-    public AccordCachingState<RoutingKey, CommandsForKey> global()
+    public AccordCacheEntry<RoutingKey, CommandsForKey> global()
     {
         checkNotInvalidated();
         return global;
@@ -113,8 +113,10 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            initialize();
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForRanges.java

@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.util.NavigableMap;
-import java.util.Objects;
-
-import accord.primitives.Ranges;
-import accord.primitives.TxnId;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import org.apache.cassandra.utils.Pair;
-
-public class AccordSafeCommandsForRanges extends ImmutableAccordSafeState<Ranges, CommandsForRanges>
-{
-    private final AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain;
-
-    public AccordSafeCommandsForRanges(Ranges ranges, AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain)
-    {
-        super(ranges);
-        this.chain = chain;
-    }
-
-    public Ranges ranges()
-    {
-        return key();
-    }
-
-    @Override
-    public void preExecute()
-    {
-        checkNotInvalidated();
-        Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>> pair = AsyncChains.getUnchecked(chain);
-        pair.left.close();
-        pair.left.get().entrySet().forEach(e -> pair.right.put(e.getKey(), e.getValue()));
-        original = CommandsForRanges.create(key, pair.right);
-    }
-
-    @Override
-    public AccordCachingState<Ranges, CommandsForRanges> global()
-    {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        AccordSafeCommandsForRanges that = (AccordSafeCommandsForRanges) o;
-        return Objects.equals(original, that.original);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(original);
-    }
-
-    @Override
-    public String toString()
-    {
-        return "AccordSafeCommandsForRange{" +
-               "chain=" + chain +
-               ", invalidated=" + invalidated +
-               ", original=" + original +
-               '}';
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordSafeState.java

@@ -18,8 +18,7 @@
 package org.apache.cassandra.service.accord;
 
 import accord.impl.SafeState;
-import accord.utils.async.AsyncChain;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
+import accord.utils.async.Cancellable;
 
 public interface AccordSafeState<K, V> extends SafeState<V>
 {
@@ -29,7 +28,7 @@ public interface AccordSafeState<K, V> extends SafeState<V>
     boolean invalidated();
     void preExecute();
 
-    AccordCachingState<K, V> global();
+    AccordCacheEntry<K, V> global();
 
     default boolean hasUpdate()
     {
@@ -46,17 +45,7 @@ public interface AccordSafeState<K, V> extends SafeState<V>
         return global().key();
     }
 
-    default Status globalStatus()
-    {
-        return global().status();
-    }
-
-    default AsyncChain<?> loading()
-    {
-        return global().loading();
-    }
-
-    default AsyncChain<?> saving()
+    default Cancellable saving()
     {
         return global().saving();
     }

src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java

@@ -31,11 +31,11 @@ import accord.primitives.Timestamp;
 public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements AccordSafeState<RoutingKey, TimestampsForKey>
 {
     private boolean invalidated;
-    private final AccordCachingState<RoutingKey, TimestampsForKey> global;
+    private final AccordCacheEntry<RoutingKey, TimestampsForKey> global;
     private TimestampsForKey original;
     private TimestampsForKey current;
 
-    public AccordSafeTimestampsForKey(AccordCachingState<RoutingKey, TimestampsForKey> global)
+    public AccordSafeTimestampsForKey(AccordCacheEntry<RoutingKey, TimestampsForKey> global)
     {
         super(global.key());
         this.global = global;
@@ -70,7 +70,7 @@ public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements
     }
 
     @Override
-    public AccordCachingState<RoutingKey, TimestampsForKey> global()
+    public AccordCacheEntry<RoutingKey, TimestampsForKey> global()
     {
         checkNotInvalidated();
         return global;
@@ -101,8 +101,10 @@ public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            initialize();
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java

@@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.PartitionUpdate.SimpleBuilder;
@@ -73,7 +72,7 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
         if (readers.isEmpty())
             return Collections.emptyList();
 
-        ColumnFamilyStore cfs = Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL);
+        ColumnFamilyStore cfs = AccordKeyspace.AccordColumnFamilyStores.journal;
         Descriptor descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables());
         SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
 

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -82,8 +82,10 @@ import accord.local.Node;
 import accord.local.Node.Id;
 import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
+import accord.local.SafeCommand;
 import accord.local.ShardDistributor.EvenSplit;
 import accord.local.cfk.CommandsForKey;
+import accord.local.cfk.SafeCommandsForKey;
 import accord.messages.Callback;
 import accord.messages.ReadData;
 import accord.messages.Reply;
@@ -127,9 +129,9 @@ import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.AccordClientRequestMetrics;
-import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.metrics.ClientRequestMetrics;
 import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessageDelivery;
@@ -181,6 +183,7 @@ import static accord.utils.Invariants.checkState;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordCommandStoreShardCount;
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
@@ -269,6 +272,9 @@ public class AccordService implements IAccordService, Shutdownable
         @Override
         public void setCacheSize(long kb) { }
 
+        @Override
+        public void setWorkingSetSize(long kb) { }
+
         @Override
         public TopologyManager topology()
         {
@@ -312,6 +318,12 @@ public class AccordService implements IAccordService, Shutdownable
             return new CompactionInfo(new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>());
         }
 
+        @Override
+        public AccordAgent agent()
+        {
+            return null;
+        }
+
         @Override
         public List<CommandStoreTxnBlockedGraph> debugTxnBlockedGraph(TxnId txnId)
         {
@@ -427,13 +439,13 @@ public class AccordService implements IAccordService, Shutdownable
         this.scheduler = new AccordScheduler();
         this.dataStore = new AccordDataStore();
         this.configuration = new AccordConfiguration(DatabaseDescriptor.getRawConfig());
-        this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal);
+        this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal, agent);
         this.node = new Node(localId,
                              messageSink,
                              configService,
                              time,
                              () -> dataStore,
-                             new KeyspaceSplitter(new EvenSplit<>(DatabaseDescriptor.getAccordShardCount(), getPartitioner().accordSplitter())),
+                             new KeyspaceSplitter(new EvenSplit<>(getAccordCommandStoreShardCount(), getPartitioner().accordSplitter())),
                              agent,
                              new DefaultRandom(),
                              scheduler,
@@ -624,13 +636,13 @@ public class AccordService implements IAccordService, Shutdownable
             AsyncResult<TxnId> asyncResult = syncPoint == null
                                                  ? Barrier.barrier(node, keysOrRanges, route, epoch, barrierType)
                                                  : Barrier.barrier(node, keysOrRanges, route, epoch, barrierType, syncPoint);
+            long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos;
+            TxnId txnId = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
             if (keysOrRanges.domain() == Key)
             {
                 PartitionKey key = (PartitionKey)keysOrRanges.get(0);
-                asyncResult.accept(txnId -> maybeSaveAccordKeyMigrationLocally(key, Epoch.create(txnId.epoch())));
+                maybeSaveAccordKeyMigrationLocally(key, Epoch.create(txnId.epoch()));
             }
-            long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos;
-            TxnId txnId = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
             ((AccordAgent) node.agent()).onSuccessfulBarrier(txnId, keysOrRanges);
             logger.debug("Completed barrier attempt in {}ms, {}ms since attempts start, barrier key: {} epoch: {} barrierType: {} isForWrite {}",
                          sw.elapsed(MILLISECONDS),
@@ -838,7 +850,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Override
     public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
     {
-        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite),
+        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite),
                              DatabaseDescriptor.getAccordBarrierRetryAttempts(),
                              DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(),
                              DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis());
@@ -847,7 +859,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Override
     public Seekables<?, ?> repairWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
     {
-        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite, allEndpoints),
+        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite, allEndpoints),
                              DatabaseDescriptor.getAccordBarrierRetryAttempts(),
                              DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(),
                              DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis());
@@ -1018,6 +1030,14 @@ public class AccordService implements IAccordService, Shutdownable
         commandStores.setCapacity(bytes);
     }
 
+    @Override
+    public void setWorkingSetSize(long kb)
+    {
+        long bytes = kb << 10;
+        AccordCommandStores commandStores = (AccordCommandStores) node.commandStores();
+        commandStores.setWorkingSetSize(bytes);
+    }
+
     @Override
     public boolean isTerminated()
     {
@@ -1126,7 +1146,7 @@ public class AccordService implements IAccordService, Shutdownable
 
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, CommandStore commandStore, TokenKey blockedBy, TxnId txnId, Timestamp executeAt)
     {
-        AsyncChain<AsyncChain<Void>> submit = commandStore.submit(PreLoadContext.contextFor(txnId, RoutingKeys.of(blockedBy.toUnseekable()), KeyHistory.COMMANDS), in -> {
+        AsyncChain<AsyncChain<Void>> submit = commandStore.submit(PreLoadContext.contextFor(txnId, RoutingKeys.of(blockedBy.toUnseekable()), KeyHistory.SYNC), in -> {
             AsyncChain<Void> chain = populate(state, (AccordSafeCommandStore) in, blockedBy, txnId, executeAt);
             return chain == null ? AsyncChains.success(null) : chain;
         });
@@ -1136,7 +1156,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Nullable
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, AccordSafeCommandStore safeStore, TxnId txnId)
     {
-        AccordSafeCommand safeCommand = safeStore.getIfLoaded(txnId);
+        SafeCommand safeCommand = safeStore.unsafeGet(txnId);
         Invariants.nonNull(safeCommand, "Txn %s is not in the cache", txnId);
         if (safeCommand.current() == null || safeCommand.current().saveStatus() == SaveStatus.Uninitialised)
             return null;
@@ -1149,7 +1169,7 @@ public class AccordService implements IAccordService, Shutdownable
         {
             if (state.knows(blockedBy)) continue;
             // need to fetch the state
-            if (safeStore.getIfLoaded(blockedBy) != null)
+            if (safeStore.ifLoadedAndInitialisedAndNotErased(blockedBy) != null)
             {
                 AsyncChain<Void> chain = populate(state, safeStore, blockedBy);
                 if (chain != null)
@@ -1164,7 +1184,7 @@ public class AccordService implements IAccordService, Shutdownable
         for (TokenKey blockedBy : cmdTxnState.blockedByKey)
         {
             if (state.keys.containsKey(blockedBy)) continue;
-            if (safeStore.getCommandsForKeyIfLoaded(blockedBy) != null)
+            if (safeStore.ifLoadedAndInitialised(blockedBy) != null)
             {
                 AsyncChain<Void> chain = populate(state, safeStore, blockedBy, txnId, safeCommand.current().executeAt());
                 if (chain != null)
@@ -1183,13 +1203,13 @@ public class AccordService implements IAccordService, Shutdownable
 
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, AccordSafeCommandStore safeStore, TokenKey pk, TxnId txnId, Timestamp executeAt)
     {
-        AccordSafeCommandsForKey commandsForKey = safeStore.getCommandsForKeyIfLoaded(pk);
+        SafeCommandsForKey commandsForKey = safeStore.ifLoadedAndInitialised(pk);
         TxnId blocking = commandsForKey.current().blockedOnTxnId(txnId, executeAt);
         if (blocking instanceof CommandsForKey.TxnInfo)
             blocking = ((CommandsForKey.TxnInfo) blocking).plainTxnId();
         state.keys.put(pk, blocking);
         if (state.txns.containsKey(blocking)) return null;
-        if (safeStore.getIfLoaded(blocking) != null) return populate(state, safeStore, blocking);
+        if (safeStore.ifLoadedAndInitialisedAndNotErased(blocking) != null) return populate(state, safeStore, blocking);
         return populate(state, safeStore.commandStore(), blocking);
     }
 
@@ -1354,6 +1374,12 @@ public class AccordService implements IAccordService, Shutdownable
         return new CompactionInfo(redundantBefores, ranges, durableBefores);
     }
 
+    @Override
+    public AccordAgent agent()
+    {
+        return (AccordAgent) node.agent();
+    }
+
     @Override
     public void awaitTableDrop(TableId id)
     {

src/java/org/apache/cassandra/service/accord/AccordStateCache.java

@@ -1,787 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.ToLongFunction;
-import java.util.stream.Stream;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.utils.IntrusiveLinkedList;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChains;
-import org.agrona.collections.Int2ObjectHashMap;
-import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
-import org.apache.cassandra.metrics.CacheAccessMetrics;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
-
-import static accord.utils.Invariants.checkState;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.SAVING;
-
-/**
- * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types.
- * </p>
- * Supports dynamic object sizes. After each acquire/free cycle, the cacheable objects size is recomputed to
- * account for data added/removed during txn processing if it's modified flag is set
- */
-public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?>> implements CacheSize
-{
-    private static final Logger logger = LoggerFactory.getLogger(AccordStateCache.class);
-
-    // Debug mode to verify that loading from journal + system tables results in
-    // functionally identical (or superceding) command to the one we've just evicted.
-    private static boolean VALIDATE_LOAD_ON_EVICT = false;
-
-    @VisibleForTesting
-    public static void validateLoadOnEvict(boolean value)
-    {
-        VALIDATE_LOAD_ON_EVICT = value;
-    }
-
-    static class Stats
-    {
-        long queries;
-        long hits;
-        long misses;
-    }
-
-    public static final class ImmutableStats
-    {
-        public final long queries;
-        public final long hits;
-        public final long misses;
-        
-        public ImmutableStats(Stats stats)
-        {
-            queries = stats.queries;
-            hits = stats.hits;
-            misses = stats.misses;
-        }
-    }
-
-    // TODO (required): cleanup on drop table, or else share between command stores
-    private Int2ObjectHashMap<Instance<?, ?, ?>> instances = new Int2ObjectHashMap<>();
-    private int nextIndex;
-
-    private final ExecutorPlus loadExecutor, saveExecutor;
-
-    private int unreferenced = 0;
-    private long maxSizeInBytes;
-    private long bytesCached = 0;
-
-    @VisibleForTesting
-    final AccordStateCacheMetrics metrics;
-    final Stats stats = new Stats();
-
-    public AccordStateCache(ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, long maxSizeInBytes, AccordStateCacheMetrics metrics)
-    {
-        this.loadExecutor = loadExecutor;
-        this.saveExecutor = saveExecutor;
-        this.maxSizeInBytes = maxSizeInBytes;
-        this.metrics = metrics;
-    }
-
-    @Override
-    public void setCapacity(long sizeInBytes)
-    {
-        maxSizeInBytes = sizeInBytes;
-        maybeEvictSomeNodes();
-    }
-
-    @Override
-    public long capacity()
-    {
-        return maxSizeInBytes;
-    }
-
-    private void unlink(AccordCachingState<?, ?> node)
-    {
-        node.unlink();
-        unreferenced--;
-    }
-
-    private void link(AccordCachingState<?, ?> node)
-    {
-        addLast(node);
-        unreferenced++;
-    }
-
-    @SuppressWarnings("unchecked")
-    private <K, V> void maybeUpdateSize(AccordCachingState<?, ?> node, ToLongFunction<?> estimator)
-    {
-        if (node.shouldUpdateSize())
-        {
-            long delta = ((AccordCachingState<K, V>) node).estimatedSizeOnHeapDelta((ToLongFunction<V>) estimator);
-            bytesCached += delta;
-            instanceForNode(node).bytesCached += delta;
-        }
-    }
-
-    /*
-     * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around
-     * for longer to maximise the chances of hitting system tables fewer times (or not at all).
-     */
-    private void maybeEvictSomeNodes()
-    {
-        if (bytesCached <= maxSizeInBytes)
-            return;
-
-        Iterator<AccordCachingState<?, ?>> iter = this.iterator();
-        while (iter.hasNext() && bytesCached > maxSizeInBytes)
-        {
-            AccordCachingState<?, ?> node = iter.next();
-            maybeEvict(node);
-        }
-    }
-
-    @VisibleForTesting
-    public boolean maybeEvict(AccordCachingState<?, ?> node)
-    {
-        checkState(node.references == 0);
-
-        /*
-         * TODO (expected, efficiency):
-         *    can this be reworked so we're not skipping unevictable nodes everytime we try to evict?
-         */
-        Status status = node.status(); // status() call completes (if completeable)
-        switch (status)
-        {
-            default: throw new IllegalStateException("Unhandled status " + status);
-            case LOADED:
-                unlink(node);
-                evict(node);
-                return true;
-            case MODIFIED:
-                // schedule a save to disk, keep linked and in the cache map
-                Instance<?, ?, ?> instance = instanceForNode(node);
-                node.save(saveExecutor, instance.saveFunction);
-                maybeUpdateSize(node, instance.heapEstimator);
-                return false;
-            case SAVING:
-                // skip over until completes to LOADED or FAILED_TO_SAVE
-                return false;
-            case FAILED_TO_SAVE:
-                // TODO (consider): panic when a save fails
-                // permanently unlink, but keep in the map
-                unlink(node);
-                return false;
-        }
-    }
-
-    private boolean isInQueue(AccordCachingState<?, ?> node)
-    {
-        return node.isLinked();
-    }
-
-    private void evict(AccordCachingState<?, ?> node)
-    {
-        if (logger.isTraceEnabled())
-            logger.trace("Evicting {} {} - {}", node.status(), node.key(), node.isLoaded() ? node.get() : null);
-
-        checkState(!isInQueue(node));
-
-        bytesCached -= node.lastQueriedEstimatedSizeOnHeap;
-        Instance<?, ?, ?> instance = instanceForNode(node);
-        instance.bytesCached -= node.lastQueriedEstimatedSizeOnHeap;
-
-        if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT)
-            instance.validateLoadEvicted(node);
-
-        AccordCachingState<?, ?> self = instances.get(node.index).cache.remove(node.key());
-        Invariants.checkState(self.references == 0);
-        checkState(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self);
-        if (instance.listeners != null)
-            instance.listeners.forEach(l -> l.onEvict((AccordCachingState) node));
-    }
-
-    public ImmutableStats stats()
-    {
-        return new ImmutableStats(stats);
-    }
-
-    private Instance<?, ?, ?> instanceForNode(AccordCachingState<?, ?> node)
-    {
-        return instances.get(node.index);
-    }
-
-    public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
-        Class<K> keyClass,
-        Class<S> valClass,
-        Function<AccordCachingState<K, V>, S> safeRefFactory,
-        Function<K, V> loadFunction,
-        Function<V, Runnable> saveFunction,
-        BiFunction<K, V, Boolean> validateFunction,
-        ToLongFunction<V> heapEstimator,
-        AccordCachingState.Factory<K, V> nodeFactory)
-    {
-        int index = ++nextIndex;
-
-        Instance<K, V, S> instance =
-            new Instance<>(index, keyClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, nodeFactory);
-
-        Int2ObjectHashMap<Instance<?, ?, ?>> newInstances = new Int2ObjectHashMap<>(instances);
-        newInstances.put(index, instance);
-        instances = newInstances;
-
-        return instance;
-    }
-
-    public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
-        Class<K> keyClass,
-        Class<S> valClass,
-        Function<AccordCachingState<K, V>, S> safeRefFactory,
-        Function<K, V> loadFunction,
-        Function<V, Runnable> saveFunction,
-        BiFunction<K, V, Boolean> validateFunction,
-        ToLongFunction<V> heapEstimator)
-    {
-        return instance(keyClass, valClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, AccordCachingState.defaultFactory());
-    }
-
-    public Collection<Instance<?, ? ,? >> instances()
-    {
-        return instances.values();
-    }
-
-    public interface Listener<K, V>
-    {
-        default void onAdd(AccordCachingState<K, V> state) {}
-        default void onRelease(AccordCachingState<K, V> state) {}
-        default void onEvict(AccordCachingState<K, V> state) {}
-    }
-
-    public class Instance<K, V, S extends AccordSafeState<K, V>> implements CacheSize, Iterable<AccordCachingState<K, V>>
-    {
-        private final int index;
-        private final Class<K> keyClass;
-        private final Function<AccordCachingState<K, V>, S> safeRefFactory;
-        private Function<K, V> loadFunction;
-        private Function<V, Runnable> saveFunction;
-        private final BiFunction<K, V, Boolean> validateFunction;
-        private final ToLongFunction<V> heapEstimator;
-        private long bytesCached;
-
-        @VisibleForTesting
-        final CacheAccessMetrics instanceMetrics;
-        private final Stats stats = new Stats();
-        private final Map<Object, AccordCachingState<?, ?>> cache = new HashMap<>();
-        private final AccordCachingState.Factory<K, V> nodeFactory;
-        private List<Listener<K, V>> listeners = null;
-
-        public Instance(
-            int index, Class<K> keyClass,
-            Function<AccordCachingState<K, V>, S> safeRefFactory,
-            Function<K, V> loadFunction,
-            Function<V, Runnable> saveFunction,
-            BiFunction<K, V, Boolean> validateFunction,
-            ToLongFunction<V> heapEstimator,
-            AccordCachingState.Factory<K, V> nodeFactory)
-        {
-            this.index = index;
-            this.keyClass = keyClass;
-            this.safeRefFactory = safeRefFactory;
-            this.loadFunction = loadFunction;
-            this.saveFunction = saveFunction;
-            this.validateFunction = validateFunction;
-            this.heapEstimator = heapEstimator;
-            this.instanceMetrics = metrics.forInstance(keyClass);
-            this.nodeFactory = nodeFactory;
-        }
-
-        public void register(Listener<K, V> l)
-        {
-            if (listeners == null)
-                listeners = new ArrayList<>();
-            listeners.add(l);
-        }
-
-        public void unregister(Listener<K, V> l)
-        {
-            if (listeners == null)
-                throw new AssertionError("No listeners exist");
-            if (!listeners.remove(l))
-                throw new AssertionError("Listener was not registered");
-            if (listeners.isEmpty())
-                listeners = null;
-        }
-
-        public Stream<AccordCachingState<K, V>> stream()
-        {
-            return cache.entrySet().stream()
-                        .filter(e -> instanceForNode(e.getValue()) == this)
-                        .map(e -> (AccordCachingState<K, V>) e.getValue());
-        }
-
-        @Override
-        public Iterator<AccordCachingState<K, V>> iterator()
-        {
-            return stream().iterator();
-        }
-
-        public S acquireOrInitialize(K key, Function<K, V> valueFactory)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-            {
-                node = nodeFactory.create(key, index);
-                node.initialize(valueFactory.apply(key));
-                cache.put(key, node);
-                if (listeners != null)
-                {
-                    AccordCachingState<K, V> finalNode = node;
-                    listeners.forEach(l -> l.onAdd(finalNode));
-                }
-            }
-            AccordCachingState<K, V> acquired = acquireExisting(node, true, null);
-            Invariants.checkState(acquired != null, "%s could not be acquired", node);
-            return safeRefFactory.apply(acquired);
-        }
-
-        public S acquireIfExists(K key)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-                return null;
-
-            return safeRefFactory.apply(acquireExisting(node, false, null));
-        }
-
-        public void maybeLoad(K key, V initial)
-        {
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-            {
-                node = nodeFactory.create(key, index);
-                node.initialize(initial);
-                Object prev = cache.put(key, node);
-                Invariants.checkState(prev == null, "%s not absent from cache: %s already present", key, node);
-                if (listeners != null)
-                {
-                    AccordCachingState<K, V> finalNode = node;
-                    listeners.forEach(l -> l.onAdd(finalNode));
-                }
-                maybeUpdateSize(node, heapEstimator);
-            }
-        }
-
-        public S acquire(K key)
-        {
-            return acquire(key, null);
-        }
-
-        public S acquireIfLoaded(K key)
-        {
-            return acquireIfLoaded(key, null);
-        }
-
-        public S acquire(K key, @Nullable ExecutorPlus loadExecutor)
-        {
-            AccordCachingState<K, V> node = acquire(key, false, loadExecutor);
-            return safeRefFactory.apply(node);
-        }
-
-        public S acquireIfLoaded(K key, @Nullable ExecutorPlus loadExecutor)
-        {
-            AccordCachingState<K, V> node = acquire(key, true, loadExecutor);
-            if (node == null)
-                return null;
-            return safeRefFactory.apply(node);
-        }
-
-        private AccordCachingState<K, V> acquire(K key, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node == null
-                 ? acquireAbsent(key, onlyIfLoaded, loadExecutor)
-                 : acquireExisting(node, onlyIfLoaded, loadExecutor);
-        }
-
-        /*
-         * Can only return a LOADING Node (or null)
-         */
-        private AccordCachingState<K, V> acquireAbsent(K key, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            incrementCacheMisses();
-            if (onlyIfLoaded)
-                return null;
-            AccordCachingState<K, V> node = nodeFactory.create(key, index);
-            if (loadExecutor == null)
-                loadExecutor = AccordStateCache.this.loadExecutor;
-            node.load(loadExecutor, loadFunction);
-            node.references++;
-
-            Object prev = cache.put(key, node);
-            Invariants.checkState(prev == null, "%s not absent from cache: %s already present", key, node);
-            if (listeners != null)
-                listeners.forEach(l -> l.onAdd(node));
-            maybeUpdateSize(node, heapEstimator);
-            metrics.objectSize.update(node.lastQueriedEstimatedSizeOnHeap);
-            maybeEvictSomeNodes();
-            return node;
-        }
-
-        /*
-         * Can't return EVICTED or INITIALIZED
-         */
-        private AccordCachingState<K, V> acquireExisting(AccordCachingState<K, V> node, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            Status status = node.status(); // status() completes
-
-            if (status.isLoaded())
-                incrementCacheHits();
-            else
-                incrementCacheMisses();
-
-            if (onlyIfLoaded && !status.isLoaded())
-                return null;
-
-            if (node.references == 0)
-            {
-                if (loadExecutor == null)
-                    loadExecutor = AccordStateCache.this.loadExecutor;
-                if (status == FAILED_TO_LOAD || status == EVICTED)
-                    node.reset().load(loadExecutor, loadFunction);
-
-                if (isInQueue(node))
-                    unlink(node);
-            }
-            node.references++;
-
-            return node;
-        }
-
-        public void release(S safeRef)
-        {
-            K key = safeRef.global().key();
-            logger.trace("Releasing resources for {}: {}", key, safeRef);
-
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-
-            checkState(safeRef.global() != null, "safeRef node is null for %s", key);
-            checkState(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node);
-            checkState(node.references > 0, "references (%d) are zero for %s (%s)", node.references, key, node);
-            checkState(!isInQueue(node));
-
-            if (safeRef.hasUpdate())
-                node.set(safeRef.current());
-            safeRef.invalidate();
-
-            maybeUpdateSize(node, heapEstimator);
-
-            if (listeners != null)
-                listeners.forEach(l -> l.onRelease(node));
-
-            if (--node.references == 0)
-            {
-                Status status = node.status(); // status() completes
-                switch (status)
-                {
-                    default: throw new IllegalStateException("Unhandled status " + status);
-                    case LOADING:
-                    case FAILED_TO_LOAD:
-                        logger.trace("Evicting {} with status {}", key, status);
-                        evict(node);
-                        break;
-                    case LOADED:
-                    case MODIFIED:
-                    case SAVING:
-                        logger.trace("Moving {} with status {} to eviction queue", key, status);
-                        link(node);
-                        break;
-                    case FAILED_TO_SAVE:
-                        break; // can never evict, so no point in adding to eviction queue either
-                }
-            }
-
-            // TODO (performance, expected): triggering on every release is potentially heavy
-            maybeEvictSomeNodes();
-        }
-
-        void validateLoadEvicted(AccordCachingState<?, ?> node)
-        {
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> state = (AccordCachingState<K, V>) node;
-            K key = state.key();
-            V evicted = state.get();
-            if (!validateFunction.apply(key, evicted))
-                throw new IllegalStateException("Reloaded value for key " + key + " is not equal to or fuller than evicted value " + evicted);
-        }
-
-        @VisibleForTesting
-        public AccordCachingState<K, V> getUnsafe(K key)
-        {
-            //noinspection unchecked
-            return (AccordCachingState<K, V>) cache.get(key);
-        }
-
-        @VisibleForTesting
-        public boolean isReferenced(K key)
-        {
-            //noinspection unchecked
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node != null && node.references > 0;
-        }
-
-        @VisibleForTesting
-        public boolean isLoaded(K key)
-        {
-            //noinspection unchecked
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node != null && node.isLoaded();
-        }
-
-        @VisibleForTesting
-        public boolean hasLoadResult(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() == LOADING;
-        }
-
-        @VisibleForTesting
-        public boolean hasSaveResult(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() == SAVING;
-        }
-
-        @VisibleForTesting
-        public void complete(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            if (node != null)
-                node.complete();
-        }
-
-        @VisibleForTesting
-        boolean keyIsReferenced(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.references > 0;
-        }
-
-        @VisibleForTesting
-        boolean keyIsCached(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() != EVICTED;
-        }
-
-        @VisibleForTesting
-        int references(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null ? node.references : 0;
-        }
-
-        private void incrementCacheQueries()
-        {
-            instanceMetrics.requests.mark();
-            metrics.requests.mark();
-            stats.queries++;
-            AccordStateCache.this.stats.queries++;
-        }
-
-        private void incrementCacheHits()
-        {
-            instanceMetrics.hits.mark();
-            metrics.hits.mark();
-            stats.hits++;
-            AccordStateCache.this.stats.hits++;
-        }
-
-        private void incrementCacheMisses()
-        {
-            instanceMetrics.misses.mark();
-            metrics.misses.mark();
-            stats.misses++;
-            AccordStateCache.this.stats.misses++;
-        }
-
-        public Stats stats()
-        {
-            return stats;
-        }
-
-        public ImmutableStats statsSnapshot()
-        {
-            return new ImmutableStats(stats);
-        }
-
-        public Stats globalStats()
-        {
-            return AccordStateCache.this.stats;
-        }
-
-        @VisibleForTesting
-        public void unsafeSetLoadFunction(Function<K, V> loadFunction)
-        {
-            this.loadFunction = loadFunction;
-        }
-
-        @VisibleForTesting
-        public void unsafeSetSaveFunction(Function<V, Runnable> saveFunction)
-        {
-            this.saveFunction = saveFunction;
-        }
-
-        @Override
-        public long capacity()
-        {
-            return AccordStateCache.this.capacity();
-        }
-
-        @Override
-        public void setCapacity(long capacity)
-        {
-            throw new UnsupportedOperationException("Capacity is shared between all instances. Please set the capacity on the global cache");
-        }
-
-        @Override
-        public int size()
-        {
-            return cache.size();
-        }
-
-        @Override
-        public long weightedSize()
-        {
-            return bytesCached;
-        }
-
-        public long globalAllocated()
-        {
-            return AccordStateCache.this.bytesCached;
-        }
-
-        public int globalReferencedEntries()
-        {
-            return AccordStateCache.this.numReferencedEntries();
-        }
-
-        public int globalUnreferencedEntries()
-        {
-            return AccordStateCache.this.numUnreferencedEntries();
-        }
-
-        @Override
-        public String toString()
-        {
-            return "Instance{" +
-                   "index=" + index +
-                   ", keyClass=" + keyClass +
-                   '}';
-        }
-    }
-
-    @VisibleForTesting
-    void unsafeClear()
-    {
-        bytesCached = 0;
-        metrics.reset();;
-        instances.values().forEach(instance -> {
-            instance.cache.forEach((k, v) -> Invariants.checkState(v.references == 0));
-            instance.cache.clear();
-            instance.bytesCached = 0;
-            instance.instanceMetrics.reset();
-        });
-        //noinspection StatementWithEmptyBody
-        while (null != poll());
-    }
-
-    @VisibleForTesting
-    AccordCachingState<?, ?> head()
-    {
-        Iterator<AccordCachingState<?, ?>> iter = iterator();
-        return iter.hasNext() ? iter.next() : null;
-    }
-
-    @VisibleForTesting
-    AccordCachingState<?, ?> tail()
-    {
-        AccordCachingState<?,?> last = null;
-        Iterator<AccordCachingState<?, ?>> iter = iterator();
-        while (iter.hasNext())
-            last = iter.next();
-        return last;
-    }
-
-    @VisibleForTesting
-    public void awaitSaveResults()
-    {
-        for (AccordCachingState<?, ?> node : this)
-            if (node.status() == SAVING)
-                AsyncChains.awaitUninterruptibly(node.saving());
-    }
-
-    private int cacheSize()
-    {
-        int size = 0;
-        for (Instance<?, ?, ?> instance : instances.values())
-            size += instance.cache.size();
-        return size;
-    }
-
-    @VisibleForTesting
-    int numReferencedEntries()
-    {
-        return cacheSize() - unreferenced;
-    }
-
-    @VisibleForTesting
-    int numUnreferencedEntries()
-    {
-        return unreferenced;
-    }
-
-    @Override
-    public int size()
-    {
-        return cacheSize();
-    }
-
-    @Override
-    public long weightedSize()
-    {
-        return bytesCached;
-    }
-}

src/java/org/apache/cassandra/service/accord/CommandsForRanges.java

@@ -60,9 +60,9 @@ public class CommandsForRanges implements CommandsSummary
         this.map = map;
     }
 
-    public static CommandsForRanges create(Ranges ranges, NavigableMap<TxnId, CommandsForRangesLoader.Summary> map)
+    public static CommandsForRanges create(Ranges ranges, NavigableMap<Timestamp, CommandsForRangesLoader.Summary> map)
     {
-        return new CommandsForRanges(ranges, (NavigableMap<Timestamp, CommandsForRangesLoader.Summary>) (NavigableMap<?, ?>) map);
+        return new CommandsForRanges(ranges, map);
     }
 
     @VisibleForTesting

src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java

@@ -19,56 +19,56 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.NavigableMap;
-import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 
 import accord.local.Command;
 import accord.local.KeyHistory;
 import accord.local.RedundantBefore;
+import accord.local.StoreParticipants;
 import accord.primitives.PartialDeps;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
 import accord.primitives.Routable.Domain;
 import accord.primitives.SaveStatus;
 import accord.primitives.Status;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.Routables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
+import accord.utils.Invariants;
 import org.agrona.collections.ObjectHashSet;
-import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.index.accord.RoutesSearcher;
+import org.apache.cassandra.service.accord.AccordCommandStore.Caches;
+import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.utils.Pair;
 
+import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
 
-public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId, Command>
+public class CommandsForRangesLoader implements AccordCache.Listener<TxnId, Command>
 {
+    private final AccordCommandStore commandStore;
     private final RoutesSearcher searcher = new RoutesSearcher();
-    private final AccordCommandStore store;
     private final NavigableMap<TxnId, Ranges> transitive = new TreeMap<>();
     private final ObjectHashSet<TxnId> cachedRangeTxns = new ObjectHashSet<>();
-    // TODO (required): make this configurable, or perhaps backed by READ stage with concurrency limit
 
-    public CommandsForRangesLoader(AccordCommandStore store)
+    public CommandsForRangesLoader(AccordCommandStore commandStore)
     {
-        this.store = store;
-        store.commandCache().register(this);
+        this.commandStore = commandStore;
+        try (ExclusiveCaches caches = commandStore.lockCaches())
+        {
+            caches.commands().register(this);
+        }
     }
 
     @Override
-    public void onAdd(AccordCachingState<TxnId, Command> state)
+    public void onAdd(AccordCacheEntry<TxnId, Command> state)
     {
         TxnId txnId = state.key();
         if (txnId.is(Domain.Range))
@@ -76,256 +76,20 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
     }
 
     @Override
-    public void onEvict(AccordCachingState<TxnId, Command> state)
+    public void onEvict(AccordCacheEntry<TxnId, Command> state)
     {
         TxnId txnId = state.key();
         if (txnId.is(Domain.Range))
             cachedRangeTxns.remove(txnId);
     }
 
-    public AsyncResult<Pair<Watcher, NavigableMap<TxnId, Summary>>> get(@Nullable TxnId primaryTxnId, KeyHistory keyHistory, Ranges ranges)
+    public Loader loader(@Nullable TxnId primaryTxnId, KeyHistory keyHistory, Ranges ranges)
     {
-        RedundantBefore redundantBefore = store.unsafeGetRedundantBefore();
+        RedundantBefore redundantBefore = commandStore.unsafeGetRedundantBefore();
         TxnId minTxnId = redundantBefore.min(ranges, e -> e.gcBefore);
-        Timestamp maxTxnId = primaryTxnId == null || keyHistory == KeyHistory.RECOVERY || !primaryTxnId.is(ExclusiveSyncPoint) ? Timestamp.MAX : primaryTxnId;
-        TxnId findAsDep = primaryTxnId != null && keyHistory == KeyHistory.RECOVERY ? primaryTxnId : null;
-        Watcher watcher = fromCache(findAsDep, ranges, minTxnId, maxTxnId, redundantBefore);
-        ImmutableMap<TxnId, Summary> before = ImmutableMap.copyOf(watcher.get());
-        return AsyncChains.ofCallable(Stage.ACCORD_RANGE_LOADER.executor(), () -> get(ranges, before, findAsDep, minTxnId, maxTxnId, redundantBefore))
-                          .map(map -> Pair.create(watcher, map), store)
-               .beginAsResult();
-    }
-
-    private NavigableMap<TxnId, Summary> get(Ranges ranges, Map<TxnId, Summary> cacheHits, @Nullable TxnId findAsDep, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-    {
-        Set<TxnId> matches = new ObjectHashSet<>();
-        for (Range range : ranges)
-            matches.addAll(intersects(range, minTxnId, maxTxnId));
-        if (matches.isEmpty())
-            return new TreeMap<>();
-        return load(ranges, cacheHits, matches, findAsDep, redundantBefore);
-    }
-
-    private Collection<TxnId> intersects(Range range, TxnId minTxnId, Timestamp maxTxnId)
-    {
-        assert range instanceof TokenRange : "Require TokenRange but given " + range.getClass();
-        Set<TxnId> intersects = searcher.intersects(store.id(), (TokenRange) range, minTxnId, maxTxnId);
-        if (!transitive.isEmpty())
-        {
-            if (intersects.isEmpty())
-                intersects = new ObjectHashSet<>();
-            for (Map.Entry<TxnId, Ranges> e : transitive.tailMap(minTxnId, true).entrySet())
-            {
-                if (e.getValue().intersects(range))
-                    intersects.add(e.getKey());
-            }
-            if (intersects.isEmpty())
-                intersects = Collections.emptySet();
-        }
-        return intersects;
-    }
-
-    public class Watcher implements AccordStateCache.Listener<TxnId, Command>, AutoCloseable
-    {
-        private final Ranges ranges;
-        private final @Nullable TxnId findAsDep;
-        private final TxnId minTxnId;
-        private final Timestamp maxTxnId;
-        private final RedundantBefore redundantBefore;
-
-        private NavigableMap<TxnId, Summary> summaries = null;
-        private Set<AccordCachingState<TxnId, Command>> needToDoubleCheck = null;
-
-        public Watcher(Ranges ranges, @Nullable TxnId findAsDep, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-        {
-            this.ranges = ranges;
-            this.findAsDep = findAsDep;
-            this.minTxnId = minTxnId;
-            this.maxTxnId = maxTxnId;
-            this.redundantBefore = redundantBefore;
-        }
-
-        public NavigableMap<TxnId, Summary> get()
-        {
-            return summaries == null ? Collections.emptyNavigableMap() : summaries;
-        }
-
-        @Override
-        public void onAdd(AccordCachingState<TxnId, Command> n)
-        {
-            if (n.key().domain() != Domain.Range)
-                return;
-
-            if (n.key().compareTo(minTxnId) < 0 || n.key().compareTo(maxTxnId) >= 0)
-                return;
-
-            AccordCachingState.State<TxnId, Command> state = n.state();
-            if (state instanceof AccordCachingState.Loading)
-            {
-                if (needToDoubleCheck == null)
-                    needToDoubleCheck = new ObjectHashSet<>();
-                needToDoubleCheck.add(n);
-                return;
-            }
-            //TODO (required): include FailedToSave?  Most likely need to, but need to improve test coverage to have failed writes
-            if (!(state instanceof AccordCachingState.Loaded
-                  || state instanceof AccordCachingState.Modified
-                  || state instanceof AccordCachingState.Saving))
-                return;
-
-            Command cmd = state.get();
-            if (cmd == null)
-                return;
-            Summary summary = create(cmd, ranges, findAsDep, redundantBefore);
-            if (summary != null)
-            {
-                if (summaries == null)
-                    summaries = new TreeMap<>();
-                summaries.put(summary.txnId, summary);
-            }
-        }
-
-        @Override
-        public void onEvict(AccordCachingState<TxnId, Command> state)
-        {
-            if (needToDoubleCheck == null)
-                return;
-            if (!needToDoubleCheck.remove(state))
-                return;
-            if (state.state() instanceof AccordCachingState.Loading)
-                return; // can't double check
-            onAdd(state);
-        }
-
-        @Override
-        public void close()
-        {
-            store.commandCache().unregister(this);
-            if (needToDoubleCheck != null)
-            {
-                Set<AccordCachingState<TxnId, Command>> copy = needToDoubleCheck;
-                needToDoubleCheck = null;
-                copy.forEach(this::onAdd);
-            }
-            needToDoubleCheck = null;
-        }
-    }
-
-    private Watcher fromCache(@Nullable TxnId findAsDep, Ranges ranges, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-    {
-        Watcher watcher = new Watcher(ranges, findAsDep, minTxnId, maxTxnId, redundantBefore);
-        for (TxnId rangeTxnId : cachedRangeTxns)
-            watcher.onAdd(store.commandCache().getUnsafe(rangeTxnId));
-        store.commandCache().register(watcher);
-        return watcher;
-    }
-
-    private NavigableMap<TxnId, Summary> load(Ranges ranges, Map<TxnId, Summary> cacheHits, Collection<TxnId> possibleTxns, @Nullable TxnId findAsDep, RedundantBefore redundantBefore)
-    {
-        //TODO (required): this logic is kinda duplicate of org.apache.cassandra.service.accord.CommandsForRange.mapReduce
-        // should figure out if this can be improved... also what is correct?
-        NavigableMap<TxnId, Summary> map = new TreeMap<>();
-        for (TxnId txnId : possibleTxns)
-        {
-            if (cacheHits.containsKey(txnId))
-                continue;
-            if (findAsDep == null)
-            {
-                SavedCommand.MinimalCommand cmd = store.loadMinimal(txnId);
-                if (cmd == null)
-                    continue; // unknown command
-                Summary summary = create(cmd, ranges, redundantBefore);
-                if (summary == null)
-                    continue;
-                map.put(txnId, summary);
-
-            }
-            else
-            {
-                Command cmd = store.loadCommand(txnId);
-                if (cmd == null)
-                    continue; // unknown command
-                Summary summary = create(cmd, ranges, findAsDep, redundantBefore);
-                if (summary == null)
-                    continue;
-                map.put(txnId, summary);
-            }
-        }
-        return map;
-    }
-
-    private static Summary create(Command cmd, Ranges cacheRanges, @Nullable TxnId findAsDep, @Nullable RedundantBefore redundantBefore)
-    {
-        //TODO (required, correctness): C* did Invalidated, accord-core did Erased... what is correct?
-        SaveStatus saveStatus = cmd.saveStatus();
-        if (saveStatus == SaveStatus.Invalidated
-            || saveStatus == SaveStatus.Erased
-            || !saveStatus.hasBeen(Status.PreAccepted))
-            return null;
-        if (cmd.partialTxn() == null)
-            return null;
-
-        Ranges keysOrRanges = cmd.participants().touches().toRanges();
-        if (keysOrRanges.domain() != Domain.Range)
-            throw new AssertionError(String.format("Txn keys are not range for %s", cmd.partialTxn()));
-        Ranges ranges = (Ranges) keysOrRanges;
-
-        ranges = ranges.slice(cacheRanges, Routables.Slice.Minimal);
-        if (ranges.isEmpty())
-            return null;
-
-        if (redundantBefore != null)
-        {
-            Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
-                if (e.gcBefore.compareTo(cmd.txnId()) < 0)
-                    return accum;
-                return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
-            }, ranges, ignore -> false);
-
-            if (newRanges.isEmpty())
-                return null;
-        }
-
-        PartialDeps partialDeps = cmd.partialDeps();
-        boolean hasAsDep = findAsDep != null && partialDeps != null && partialDeps.rangeDeps.intersects(findAsDep, ranges);
-        return new Summary(cmd.txnId(), cmd.executeAt(), saveStatus, ranges, findAsDep, hasAsDep);
-    }
-
-    private static Summary create(SavedCommand.MinimalCommand cmd, Ranges cacheRanges, @Nullable RedundantBefore redundantBefore)
-    {
-        //TODO (required, correctness): C* did Invalidated, accord-core did Erased... what is correct?
-        SaveStatus saveStatus = cmd.saveStatus;
-        if (saveStatus == null
-            || saveStatus == SaveStatus.Invalidated
-            || saveStatus == SaveStatus.Erased
-            || !saveStatus.hasBeen(Status.PreAccepted))
-            return null;
-
-        if (cmd.participants == null)
-            return null;
-
-        Ranges keysOrRanges = cmd.participants.touches().toRanges();
-        if (keysOrRanges.domain() != Domain.Range)
-            throw new AssertionError(String.format("Txn keys are not range for %s", cmd.participants));
-        Ranges ranges = (Ranges) keysOrRanges;
-
-        ranges = ranges.slice(cacheRanges, Routables.Slice.Minimal);
-        if (ranges.isEmpty())
-            return null;
-
-        if (redundantBefore != null)
-        {
-            Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
-                if (e.gcBefore.compareTo(cmd.txnId) < 0)
-                    return accum;
-                return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
-            }, ranges, ignore -> false);
-
-            if (newRanges.isEmpty())
-                return null;
-        }
-
-        return new Summary(cmd.txnId, cmd.executeAt, saveStatus, ranges, null, false);
+        Timestamp maxTxnId = primaryTxnId == null || keyHistory == KeyHistory.RECOVER || !primaryTxnId.is(ExclusiveSyncPoint) ? Timestamp.MAX : primaryTxnId;
+        TxnId findAsDep = primaryTxnId != null && keyHistory == KeyHistory.RECOVER ? primaryTxnId : null;
+        return new Loader(ranges, redundantBefore, minTxnId, maxTxnId, findAsDep);
     }
 
     public void mergeTransitive(TxnId txnId, Ranges ranges, BiFunction<? super Ranges, ? super Ranges, ? extends Ranges> remappingFunction)
@@ -370,7 +134,7 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
 
         public Summary slice(Ranges slice)
         {
-            return new Summary(txnId, executeAt, saveStatus, ranges.slice(slice, Routables.Slice.Minimal), findAsDep, hasAsDep);
+            return new Summary(txnId, executeAt, saveStatus, ranges == null ? null : ranges.slice(slice, Minimal), findAsDep, hasAsDep);
         }
 
         @Override
@@ -386,4 +150,150 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
                    '}';
         }
     }
+
+    public class Loader
+    {
+        final Ranges searchRanges;
+        final RedundantBefore redundantBefore;
+        final TxnId minTxnId;
+        final Timestamp maxTxnId;
+        @Nullable final TxnId findAsDep;
+
+        public Loader(Ranges searchRanges, RedundantBefore redundantBefore, TxnId minTxnId, Timestamp maxTxnId, @Nullable TxnId findAsDep)
+        {
+            this.searchRanges = searchRanges;
+            this.redundantBefore = redundantBefore;
+            this.minTxnId = minTxnId;
+            this.maxTxnId = maxTxnId;
+            this.findAsDep = findAsDep;
+        }
+
+        public Collection<TxnId> intersects()
+        {
+            ObjectHashSet<TxnId> txnIds = new ObjectHashSet<>();
+            for (Range range : searchRanges)
+            {
+                searcher.intersects(commandStore.id(), (TokenRange) range, minTxnId, maxTxnId, txnIds::add);
+            }
+            if (!transitive.isEmpty())
+            {
+                for (Map.Entry<TxnId, Ranges> e : transitive.tailMap(minTxnId, true).entrySet())
+                {
+                    if (e.getValue().intersects(searchRanges))
+                        txnIds.add(e.getKey());
+                }
+            }
+            return txnIds;
+        }
+
+        public void forEachInCache(Consumer<Summary> forEach, Caches caches)
+        {
+            for (TxnId txnId : cachedRangeTxns)
+            {
+                AccordCacheEntry<TxnId, Command> state = caches.commands().getUnsafe(txnId);
+                Summary summary = from(state);
+                if (summary != null)
+                    forEach.accept(summary);
+            }
+        }
+
+        public Summary load(TxnId txnId)
+        {
+            if (findAsDep == null)
+            {
+                SavedCommand.MinimalCommand cmd = commandStore.loadMinimal(txnId);
+                return cmd == null ? null : from(cmd);
+            }
+            else
+            {
+                Command cmd = commandStore.loadCommand(txnId);
+                return cmd == null ? null : from(cmd);
+            }
+        }
+
+        public Summary from(AccordCacheEntry<TxnId, Command> state)
+        {
+            if (state.key().domain() != Domain.Range)
+                return null;
+
+            switch (state.status())
+            {
+                default: throw new AssertionError("Unhandled status: " + state.status());
+                case LOADING:
+                case WAITING_TO_LOAD:
+                case UNINITIALIZED:
+                    return null;
+
+                case LOADED:
+                case MODIFIED:
+                case SAVING:
+                case FAILED_TO_SAVE:
+            }
+
+            TxnId txnId = state.key();
+            if (!txnId.isVisible() || txnId.compareTo(minTxnId) < 0 || txnId.compareTo(maxTxnId) >= 0)
+                return null;
+
+            Command command = state.getExclusive();
+            if (command == null)
+                return null;
+            return from(command);
+        }
+
+        public Summary from(Command cmd)
+        {
+            return from(cmd.txnId(), cmd.executeAt(), cmd.saveStatus(), cmd.participants(), cmd.partialDeps());
+        }
+
+        public Summary from(SavedCommand.MinimalCommand cmd)
+        {
+            Invariants.checkState(findAsDep == null);
+            return from(cmd.txnId, cmd.executeAt, cmd.saveStatus, cmd.participants, null);
+        }
+
+        private Summary from(TxnId txnId, Timestamp executeAt, SaveStatus saveStatus, StoreParticipants participants, @Nullable PartialDeps partialDeps)
+        {
+            if (saveStatus == SaveStatus.Invalidated
+                || saveStatus == SaveStatus.Erased
+                || !saveStatus.hasBeen(Status.PreAccepted))
+                return null;
+
+            if (participants == null)
+                return null;
+
+            Ranges keysOrRanges = participants.touches().toRanges();
+            if (keysOrRanges.domain() != Domain.Range)
+                throw new AssertionError(String.format("Txn keys are not range for %s", participants));
+            Ranges ranges = keysOrRanges;
+
+            ranges = ranges.slice(searchRanges, Minimal);
+            if (ranges.isEmpty())
+                return null;
+
+            if (redundantBefore != null)
+            {
+                Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
+                    if (e.gcBefore.compareTo(txnId) < 0)
+                        return accum;
+                    return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
+                }, ranges, ignore -> false);
+
+                if (newRanges.isEmpty())
+                    return null;
+
+                ranges = newRanges;
+            }
+
+            Invariants.checkState(partialDeps != null || findAsDep == null || !saveStatus.known.deps.hasProposedOrDecidedDeps());
+            boolean hasAsDep = false;
+            if (partialDeps != null)
+            {
+                Ranges depRanges = partialDeps.rangeDeps.ranges(txnId);
+                if (depRanges != null && depRanges.containsAll(ranges))
+                    hasAsDep = true;
+            }
+
+            return new Summary(txnId, executeAt, saveStatus, ranges, findAsDep, hasAsDep);
+        }
+    }
 }

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -51,6 +51,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordScheduler;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.tcm.Epoch;
@@ -114,6 +115,7 @@ public interface IAccordService
     long currentEpoch();
 
     void setCacheSize(long kb);
+    void setWorkingSetSize(long kb);
 
     TopologyManager topology();
 
@@ -152,6 +154,8 @@ public interface IAccordService
      */
     CompactionInfo getCompactionInfo();
 
+    AccordAgent agent();
+
     default Id nodeId() { throw new UnsupportedOperationException(); }
 
     List<CommandStoreTxnBlockedGraph> debugTxnBlockedGraph(TxnId txnId);

src/java/org/apache/cassandra/service/accord/IJournal.java

@@ -39,7 +39,7 @@ public interface IJournal
     NavigableMap<Timestamp, Ranges> loadSafeToRead(int commandStoreId);
     CommandStores.RangesForEpoch.Snapshot loadRangesForEpoch(int commandStoreId);
 
-    void appendCommand(int store, SavedCommand.DiffWriter value, Runnable onFlush);
+    void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush);
     Persister<DurableBefore, DurableBefore> durableBeforePersister();
     void persistStoreState(int store,
                            // TODO: this class should not live under ASCS

src/java/org/apache/cassandra/service/accord/JournalKey.java

@@ -92,6 +92,14 @@ public final class JournalKey
             serializeTxnId(key.id, out);
         }
 
+        @Override
+        public void serialize(JournalKey key, ByteBuffer out, int userVersion) throws IOException
+        {
+            out.putInt(key.commandStoreId);
+            out.put((byte) key.type.id);
+            serializeTxnId(key.id, out);
+        }
+
         private void serialize(JournalKey key, byte[] out)
         {
             ByteArrayUtil.putInt(out, CS_ID_OFFSET, key.commandStoreId);
@@ -117,6 +125,15 @@ public final class JournalKey
             return new JournalKey(txnId, Type.fromId(type), commandStoreId);
         }
 
+        @Override
+        public JournalKey deserialize(ByteBuffer buffer, int userVersion)
+        {
+            int commandStoreId = buffer.getInt();
+            int type = buffer.get();
+            TxnId txnId = deserializeTxnId(buffer);
+            return new JournalKey(txnId, Type.fromId(type), commandStoreId);
+        }
+
         private void serializeTxnId(TxnId txnId, DataOutputPlus out) throws IOException
         {
             out.writeLong(txnId.msb);
@@ -132,6 +149,14 @@ public final class JournalKey
             return TxnId.fromBits(msb, lsb, new Id(nodeId));
         }
 
+        private TxnId deserializeTxnId(ByteBuffer in)
+        {
+            long msb = in.getLong();
+            long lsb = in.getLong();
+            int nodeId = in.getInt();
+            return TxnId.fromBits(msb, lsb, new Id(nodeId));
+        }
+
         private void serializeTxnId(TxnId txnId, byte[] out)
         {
             ByteArrayUtil.putLong(out, MSB_OFFSET, txnId.msb);
@@ -139,6 +164,13 @@ public final class JournalKey
             ByteArrayUtil.putInt(out, NODE_OFFSET, txnId.node.id);
         }
 
+        private void serializeTxnId(TxnId txnId, ByteBuffer out)
+        {
+            out.putLong(txnId.msb);
+            out.putLong(txnId.lsb);
+            out.putInt(txnId.node.id);
+        }
+
         private TxnId deserializeTxnId(ByteBuffer buffer, int position)
         {
             long msb = buffer.getLong(position + MSB_OFFSET);

src/java/org/apache/cassandra/service/accord/SavedCommand.java

@@ -20,11 +20,13 @@ package org.apache.cassandra.service.accord;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.EnumSet;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import accord.api.Agent;
 import accord.api.Result;
 import accord.local.Cleanup;
 import accord.local.Command;
@@ -89,31 +91,24 @@ public class SavedCommand
     }
 
     // TODO: maybe rename this and enclosing classes?
-    public static class DiffWriter implements Journal.Writer
+    public static class Writer implements Journal.Writer
     {
-        private final Command before;
         private final Command after;
         private final TxnId txnId;
+        private final int flags;
 
-        // TODO: improve encapsulationd
         @VisibleForTesting
-        public DiffWriter(Command before, Command after)
+        public Writer(Command after, int flags)
         {
-            this(after.txnId(), before, after);
+            this(after.txnId(), after, flags);
         }
 
         @VisibleForTesting
-        public DiffWriter(TxnId txnId, Command before, Command after)
+        public Writer(TxnId txnId, Command after, int flags)
         {
             this.txnId = txnId;
-            this.before = before;
             this.after = after;
-        }
-
-        @VisibleForTesting
-        public Command before()
-        {
-            return before;
+            this.flags = flags;
         }
 
         @VisibleForTesting
@@ -124,7 +119,7 @@ public class SavedCommand
 
         public void write(DataOutputPlus out, int userVersion) throws IOException
         {
-            serialize(before, after, out, userVersion);
+            serialize(after, flags, out, userVersion);
         }
 
         public TxnId key()
@@ -133,30 +128,37 @@ public class SavedCommand
         }
     }
 
-    public static ByteBuffer asSerializedDiff(Command after, int userVersion) throws IOException
+    public static @Nullable ByteBuffer asSerializedDiff(Command before, Command after, int userVersion) throws IOException
     {
         try (DataOutputBuffer out = new DataOutputBuffer())
         {
-            diff(null, after).write(out, userVersion);
+            Writer writer = diff(before, after);
+            if (writer == null)
+                return null;
+
+            writer.write(out, userVersion);
             return out.asNewBuffer();
         }
     }
 
     @Nullable
-    public static DiffWriter diff(Command original, Command current)
+    public static Writer diff(Command original, Command current)
     {
         if (original == current
             || current == null
-            || current.saveStatus() == SaveStatus.Uninitialised
-            || !anyFieldChanged(original, current))
+            || current.saveStatus() == SaveStatus.Uninitialised)
             return null;
-        return new SavedCommand.DiffWriter(original, current);
+
+        int flags = validateFlags(getFlags(original, current));
+        if (!anyFieldChanged(flags))
+            return null;
+
+        return new Writer(current, flags);
     }
 
     // TODO (required): calculate flags once
-    private static boolean anyFieldChanged(Command before, Command after)
+    private static boolean anyFieldChanged(int flags)
     {
-        int flags = validateFlags(getFlags(before, after));
         return (flags >>> 16) != 0;
     }
 
@@ -166,9 +168,9 @@ public class SavedCommand
         return flags;
     }
     
-    public static void serialize(Command before, Command after, DataOutputPlus out, int userVersion) throws IOException
+    public static void serialize(Command after, int flags, DataOutputPlus out, int userVersion) throws IOException
     {
-        int flags = validateFlags(getFlags(before, after));
+        Invariants.checkState(flags != 0);
         out.writeInt(flags);
 
         int iterable = toIterableSetFields(flags);
@@ -230,7 +232,7 @@ public class SavedCommand
     }
 
     @VisibleForTesting
-    static int getFlags(Command before, Command after)
+    public static int getFlags(Command before, Command after)
     {
         int flags = 0;
 
@@ -296,6 +298,17 @@ public class SavedCommand
         return (oldFlags & (0x10000 << field.ordinal())) != 0;
     }
 
+    static EnumSet<Fields> getFieldsChanged(int flags)
+    {
+        EnumSet<Fields> fields = EnumSet.noneOf(Fields.class);
+        for (Fields field : Fields.FIELDS)
+        {
+            if ((flags & (0x10000 << field.ordinal())) != 0)
+                fields.add(field);
+        }
+        return fields;
+    }
+
     static int toIterableSetFields(int flags)
     {
         return flags >>> 16;
@@ -536,7 +549,7 @@ public class SavedCommand
             return count;
         }
 
-        public Cleanup shouldCleanup(RedundantBefore redundantBefore, DurableBefore durableBefore)
+        public Cleanup shouldCleanup(Agent agent, RedundantBefore redundantBefore, DurableBefore durableBefore)
         {
             if (!nextCalled)
                 return NO;
@@ -544,7 +557,7 @@ public class SavedCommand
             if (saveStatus == null || participants == null)
                 return Cleanup.NO;
 
-            Cleanup cleanup = Cleanup.shouldCleanupPartial(txnId, saveStatus, durability, participants, redundantBefore, durableBefore);
+            Cleanup cleanup = Cleanup.shouldCleanupPartial(agent, txnId, saveStatus, durability, participants, redundantBefore, durableBefore);
             if (this.cleanup != null && this.cleanup.compareTo(cleanup) > 0)
                 cleanup = this.cleanup;
             return cleanup;
@@ -661,6 +674,7 @@ public class SavedCommand
         public void serialize(DataOutputPlus out, int userVersion) throws IOException
         {
             Invariants.checkState(mask == 0);
+            Invariants.checkState(flags != 0);
             out.writeInt(validateFlags(flags));
 
             int iterable = toIterableSetFields(flags);
@@ -718,12 +732,11 @@ public class SavedCommand
             }
         }
 
-        // TODO: we seem to be writing some form of empty transaction
-        @SuppressWarnings({ "rawtypes", "unchecked" })
         public void deserializeNext(DataInputPlus in, int userVersion) throws IOException
         {
             Invariants.checkState(txnId != null);
-            final int flags = in.readInt();
+            int flags = in.readInt();
+            Invariants.checkState(flags != 0);
             nextCalled = true;
             count++;
 

src/java/org/apache/cassandra/service/accord/api/AccordAgent.java

@@ -57,6 +57,7 @@ import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static accord.primitives.Routable.Domain.Key;
+import static accord.utils.Invariants.illegalState;
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -185,16 +186,10 @@ public class AccordAgent implements Agent
         return AccordMetrics.Listener.instance;
     }
 
-    @Override
-    public long replyTimeout(ReplyContext replyContext, TimeUnit units)
-    {
-        return Math.max(1, units.convert(((ResponseContext)replyContext).expiresAtNanos() - Clock.Global.nanoTime(), NANOSECONDS));
-    }
-
     @Override
     public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount)
     {
-        SafeCommand safeCommand = safeStore.ifInitialised(txnId);
+        SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId);
         Invariants.nonNull(safeCommand);
 
         Command command = safeCommand.current();
@@ -250,4 +245,17 @@ public class AccordAgent implements Agent
         // TODO (expected): integrate with contention backoff
         return units.convert((1L << Math.min(retryCount, 4)), SECONDS);
     }
+
+    @Override
+    public long expiresAt(ReplyContext replyContext, TimeUnit unit)
+    {
+        return unit.convert(((ResponseContext)replyContext).expiresAtNanos(), NANOSECONDS);
+    }
+
+    @Override
+    public void onViolation(String message)
+    {
+        try { throw illegalState(message); }
+        catch (Throwable t) { logger.error("Consistency violation", t); }
+    }
 }

src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java

@@ -19,7 +19,6 @@
 package org.apache.cassandra.service.accord.api;
 
 import java.io.IOException;
-import java.util.Objects;
 
 import accord.primitives.RoutableKey;
 import org.apache.cassandra.dht.Token;
@@ -65,7 +64,7 @@ public abstract class AccordRoutableKey implements RoutableKey
     @Override
     public int hashCode()
     {
-        return Objects.hash(table, token().tokenHash());
+        return table.hashCode() * 31 + token().tokenHash();
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java

@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.TreeMap;
 
 import accord.api.Key;
@@ -112,7 +111,7 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         @Override
         public int hashCode()
         {
-            return Objects.hash(table, isMin);
+            return table.hashCode() * (isMin ? 31 : 1);
         }
 
         @Override
@@ -462,5 +461,11 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         {
             return subSplitter.splitRange(range, from, to, numSplits);
         }
+
+        @Override
+        public int numberOfSplitsPossible(Range range)
+        {
+            return subSplitter.numberOfSplitsPossible(range);
+        }
     }
 }

src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java

@@ -30,7 +30,7 @@ import org.apache.cassandra.concurrent.Shutdownable;
 
 public class AccordScheduler implements Scheduler, Shutdownable
 {
-    private final ScheduledExecutorPlus scheduledExecutor = ExecutorFactory.Global.executorFactory().scheduled("AccordScheduled");
+    private final ScheduledExecutorPlus scheduledExecutor = ExecutorFactory.Global.executorFactory().scheduled(false, "AccordScheduled");
 
     private static class ScheduledFutureWrapper implements Scheduled
     {

src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java

@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord.async;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.api.RoutingKey;
-import accord.local.KeyHistory;
-import accord.local.PreLoadContext;
-import accord.local.cfk.CommandsForKey;
-import accord.primitives.AbstractKeys;
-import accord.primitives.AbstractRanges;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import accord.utils.async.Observable;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.service.accord.AccordCachingState;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForRanges;
-import org.apache.cassandra.service.accord.AccordSafeState;
-import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.CommandsForRangesLoader;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.utils.NoSpamLogger;
-import org.apache.cassandra.utils.Pair;
-
-public class AsyncLoader
-{
-    private static final Logger logger = LoggerFactory.getLogger(AsyncLoader.class);
-    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.MINUTES);
-
-    enum State
-    {
-        INITIALIZED,
-        SETUP,
-        LOADING,
-        FINISHED
-    }
-
-    private State state = State.INITIALIZED;
-    private final AccordCommandStore commandStore;
-
-    private final Iterable<TxnId> txnIds;
-    private final Unseekables<?> keysOrRanges;
-    private final KeyHistory keyHistory;
-
-    protected AsyncResult<?> readResult;
-
-    public AsyncLoader(AccordCommandStore commandStore, Iterable<TxnId> txnIds, Unseekables<?> keysOrRanges, KeyHistory keyHistory)
-    {
-        this.commandStore = commandStore;
-        this.txnIds = txnIds;
-        this.keysOrRanges = keysOrRanges;
-        this.keyHistory = keyHistory;
-    }
-
-    protected static Iterable<TxnId> txnIds(PreLoadContext context)
-    {
-        TxnId primaryid = context.primaryTxnId();
-        Collection<TxnId> additionalIds = context.additionalTxnIds();
-        if (primaryid == null) return additionalIds;
-        if (additionalIds.isEmpty()) return Collections.singleton(primaryid);
-        return Iterables.concat(Collections.singleton(primaryid), additionalIds);
-    }
-
-    private static <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReadsForKey(K key,
-                                                                                                Map<K, S> context,
-                                                                                                AccordStateCache.Instance<K, V, S> cache,
-                                                                                                List<AsyncChain<?>> listenChains)
-    {
-        referenceAndAssembleReadsForKey(key, context, cache, listenChains, null);
-    }
-
-    private static <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReadsForKey(K key,
-                                                                                                Map<K, S> context,
-                                                                                                AccordStateCache.Instance<K, V, S> cache,
-                                                                                                List<AsyncChain<?>> listenChains,
-                                                                                                @Nullable ExecutorPlus loadExecutor)
-    {
-        S safeRef = cache.acquire(key, loadExecutor);
-        if (context.putIfAbsent(key, safeRef) != null)
-        {
-            noSpamLogger.warn("Context {} contained key {} more than once", context, key);
-            cache.release(safeRef);
-            return;
-        }
-        AccordCachingState.Status status = safeRef.globalStatus(); // globalStatus() completes
-        switch (status)
-        {
-            default: throw new IllegalStateException("Unhandled global state: " + status);
-            case LOADING:
-                listenChains.add(safeRef.loading());
-                break;
-            case SAVING:
-                // make sure we work with a completed state that supports get() and set()
-                listenChains.add(safeRef.saving());
-                break;
-            case LOADED:
-            case MODIFIED:
-            case FAILED_TO_SAVE:
-                break;
-            case FAILED_TO_LOAD:
-                // TODO (required): if this triggers, we trigger some other illegal state in cache management
-                throw new RuntimeException(safeRef.failure());
-        }
-    }
-
-    private void referenceAndAssembleReadsForKey(RoutingKey key,
-                                                 AsyncOperation.Context context,
-                                                 List<AsyncChain<?>> listenChains)
-    {
-        referenceAndAssembleReadsForKey(key, context, listenChains, null);
-    }
-
-    private void referenceAndAssembleReadsForKey(RoutingKey key,
-                                                 AsyncOperation.Context context,
-                                                 List<AsyncChain<?>> listenChains,
-                                                 @Nullable ExecutorPlus loadExecutor)
-    {
-        // recovery operations also need the deps data for their preaccept logic
-        switch (keyHistory)
-        {
-            case TIMESTAMPS:
-                referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains, loadExecutor);
-                break;
-            case COMMANDS:
-            case RECOVERY:
-                referenceAndAssembleReadsForKey(key, context.commandsForKey, commandStore.commandsForKeyCache(), listenChains, loadExecutor);
-            case NONE:
-                break;
-            default: throw new IllegalArgumentException("Unhandled keyhistory: " + keyHistory);
-        }
-    }
-
-    private <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReads(Iterable<? extends K> keys,
-                                                                                   Map<K, S> context,
-                                                                                   AccordStateCache.Instance<K, V, S> cache,
-                                                                                   List<AsyncChain<?>> listenChains)
-    {
-        keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, cache, listenChains));
-    }
-
-    private AsyncResult<?> referenceAndDispatchReads(@Nullable TxnId primaryTxnId, AsyncOperation.Context context)
-    {
-        List<AsyncChain<?>> chains = new ArrayList<>();
-
-        referenceAndAssembleReads(txnIds, context.commands, commandStore.commandCache(), chains);
-
-        switch (keysOrRanges.domain())
-        {
-            case Key:
-                AbstractKeys<RoutingKey> keys = (AbstractKeys<RoutingKey>) keysOrRanges;
-                keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
-                break;
-            case Range:
-                chains.add(referenceAndDispatchReadsForRange(primaryTxnId, context));
-                break;
-            default:
-                throw new UnsupportedOperationException("Unable to process keys of " + keysOrRanges.domain());
-        }
-
-        return !chains.isEmpty() ? AsyncChains.reduce(chains, (a, b) -> null).beginAsResult() : null;
-    }
-
-    private AsyncChain<?> referenceAndDispatchReadsForRange(@Nullable TxnId primaryTxnId, AsyncOperation.Context context)
-    {
-        if (keyHistory == KeyHistory.NONE)
-            return AsyncChains.success(null);
-
-        Ranges ranges = ((AbstractRanges) keysOrRanges).toRanges();
-
-        List<AsyncChain<?>> root = new ArrayList<>(ranges.size() + 1);
-        class Watcher implements AccordStateCache.Listener<RoutingKey, CommandsForKey>
-        {
-            // TODO (required): streams prohibited in hot path
-            private final Set<TokenKey> cached = commandStore.commandsForKeyCache().stream()
-                                                                 .map(n -> (TokenKey) n.key())
-                                                                 .filter(ranges::contains)
-                                                                 .collect(Collectors.toSet());
-
-            @Override
-            public void onAdd(AccordCachingState<RoutingKey, CommandsForKey> state)
-            {
-                TokenKey pk = (TokenKey) state.key();
-                if (ranges.contains(pk))
-                    cached.add(pk);
-            }
-        }
-
-        // TODO (required): this needs to be optimised (e.g. to not load redundant commands, but maybe to be avoided altogether with async evaluation)
-        Watcher watcher = new Watcher();
-        commandStore.commandsForKeyCache().register(watcher);
-        root.add(findOverlappingKeys(ranges).flatMap(keys -> {
-            commandStore.commandsForKeyCache().unregister(watcher);
-            if (keys.isEmpty() && watcher.cached.isEmpty())
-                return AsyncChains.success(null);
-            Set<? extends RoutingKey> set = ImmutableSet.<RoutingKey>builder().addAll(watcher.cached).addAll(keys).build();
-            List<AsyncChain<?>> chains = new ArrayList<>();
-            set.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains, Stage.ACCORD_RANGE_LOADER.executor()));
-            return chains.isEmpty() ? AsyncChains.success(null) : AsyncChains.reduce(chains, (a, b) -> null);
-        }, commandStore));
-
-        AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain = commandStore.diskCommandsForRanges().get(primaryTxnId, keyHistory, ranges);
-        root.add(chain);
-        context.commandsForRanges = new AccordSafeCommandsForRanges(ranges, chain);
-
-        return AsyncChains.all(root);
-    }
-
-    private AsyncChain<List<? extends RoutingKey>> findOverlappingKeys(Ranges ranges)
-    {
-        if (ranges.isEmpty())
-        {
-            // During topology changes some shards may be included with empty ranges
-            return AsyncChains.success(Collections.emptyList());
-        }
-
-        List<AsyncChain<List<TokenKey>>> chains = new ArrayList<>(ranges.size());
-        for (Range range : ranges)
-            chains.add(findOverlappingKeys(range));
-        return AsyncChains.reduce(chains, (a, b) -> ImmutableList.<RoutingKey>builderWithExpectedSize(a.size() + b.size()).addAll(a).addAll(b).build());
-    }
-
-    private AsyncChain<List<TokenKey>> findOverlappingKeys(Range range)
-    {
-        // save to a variable as java gets confused when `.map` is called on the result of asChain
-        AsyncChain<List<TokenKey>> map = Observable.asChain(callback ->
-                                                               AccordKeyspace.findAllKeysBetween(commandStore.id(),
-                                                                                                 (AccordRoutingKey) range.start(), range.startInclusive(),
-                                                                                                 (AccordRoutingKey) range.end(), range.endInclusive(),
-                                                                                                 callback),
-                                                               Collectors.toList());
-        return map.map(ImmutableList::copyOf);
-    }
-
-    @VisibleForTesting
-    void state(State state)
-    {
-        this.state = state;
-    }
-
-    public boolean load(@Nullable TxnId primaryTxnId, AsyncOperation.Context context, BiConsumer<Object, Throwable> callback)
-    {
-        logger.trace("Running load for {} with state {}: {} {}", callback, state, txnIds, keysOrRanges);
-        commandStore.checkInStoreThread();
-        switch (state)
-        {
-            case INITIALIZED:
-                state(State.SETUP);
-
-            case SETUP:
-                readResult = referenceAndDispatchReads(primaryTxnId, context);
-                state(State.LOADING);
-
-            case LOADING:
-                if (readResult != null)
-                {
-                    if (readResult.isSuccess())
-                    {
-                        logger.trace("Read result succeeded for {}", callback);
-                        readResult = null;
-                    }
-                    else
-                    {
-                        logger.trace("Adding callback for read result: {}", callback);
-                        readResult.addCallback(callback, commandStore.executor());
-                        break;
-                    }
-                }
-                state(State.FINISHED);
-
-            case FINISHED:
-                break;
-
-            default:
-                throw new IllegalStateException("Unexpected state: " + state);
-        }
-
-        if (logger.isTraceEnabled())
-            logger.trace("Exiting load for {} with state {}: {} {}", callback, state, txnIds, keysOrRanges);
-
-        return state == State.FINISHED;
-    }
-}

src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java

@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord.async;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import accord.api.RoutingKey;
-import accord.local.Command;
-import accord.local.CommandStore;
-import accord.local.PreLoadContext;
-import accord.local.SafeCommandStore;
-import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.Cancellable;
-import org.agrona.collections.Object2ObjectHashMap;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeCommandStore;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForRanges;
-import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
-import org.apache.cassandra.service.accord.SavedCommand;
-import org.apache.cassandra.utils.concurrent.Condition;
-
-import static org.apache.cassandra.service.accord.async.AsyncLoader.txnIds;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.COMPLETING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FAILED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FINISHED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.INITIALIZED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.LOADING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.PREPARING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.RUNNING;
-
-public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements Runnable, Function<SafeCommandStore, R>, Cancellable
-{
-    private static final Logger logger = LoggerFactory.getLogger(AsyncOperation.class);
-
-    private static class LoggingProps
-    {
-        private static final String COMMAND_STORE = "command_store";
-        private static final String ASYNC_OPERATION = "async_op";
-    }
-
-    static class Context
-    {
-        final Object2ObjectHashMap<TxnId, AccordSafeCommand> commands = new Object2ObjectHashMap<>();
-        final Object2ObjectHashMap<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey = new Object2ObjectHashMap<>();
-        final Object2ObjectHashMap<RoutingKey, AccordSafeCommandsForKey> commandsForKey = new Object2ObjectHashMap<>();
-        @Nullable
-        AccordSafeCommandsForRanges commandsForRanges = null;
-
-        void releaseResources(AccordCommandStore commandStore)
-        {
-            // TODO (expected): we should destructively iterate to avoid invoking second time in fail; or else read and set to null
-            commands.forEach((k, v) -> commandStore.commandCache().release(v));
-            commands.clear();
-            timestampsForKey.forEach((k, v) -> commandStore.timestampsForKeyCache().release(v));
-            timestampsForKey.clear();
-            commandsForKey.forEach((k, v) -> commandStore.commandsForKeyCache().release(v));
-            commandsForKey.clear();
-        }
-
-        void revertChanges()
-        {
-            commands.forEach((k, v) -> v.revert());
-            timestampsForKey.forEach((k, v) -> v.revert());
-            commandsForKey.forEach((k, v) -> v.revert());
-            if (commandsForRanges != null)
-                commandsForRanges.revert();
-        }
-    }
-
-    enum State
-    {
-        INITIALIZED, LOADING, PREPARING, RUNNING, COMPLETING, AWAITING_FLUSH, FINISHED, FAILED;
-
-        boolean isComplete()
-        {
-            return this == FINISHED || this == FAILED;
-        }
-    }
-
-    private State state = INITIALIZED;
-    private final AccordCommandStore commandStore;
-    private final PreLoadContext preLoadContext;
-    private final Context context = new Context();
-    private AccordSafeCommandStore safeStore;
-    private final AsyncLoader loader;
-    private R result;
-    private final String loggingId;
-    private BiConsumer<? super R, Throwable> callback;
-
-    private List<Command> sanityCheck = null;
-
-    private void setLoggingIds()
-    {
-        MDC.put(LoggingProps.COMMAND_STORE, commandStore.loggingId);
-        MDC.put(LoggingProps.ASYNC_OPERATION, loggingId);
-    }
-
-    private void clearLoggingIds()
-    {
-        MDC.remove(LoggingProps.COMMAND_STORE);
-        MDC.remove(LoggingProps.ASYNC_OPERATION);
-    }
-
-    public AsyncOperation(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-    {
-        this.loggingId = "0x" + Integer.toHexString(System.identityHashCode(this));
-        this.commandStore = commandStore;
-        this.preLoadContext = preLoadContext;
-        this.loader = createAsyncLoader(commandStore, preLoadContext);
-
-        if (logger.isTraceEnabled())
-        {
-            setLoggingIds();
-            logger.trace("Created {} on {}", this, commandStore);
-            clearLoggingIds();
-        }
-    }
-
-    @Override
-    public String toString()
-    {
-        return "AsyncOperation{" + state + "}-" + loggingId;
-    }
-
-    AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-    {
-        return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys(), preLoadContext.keyHistory());
-    }
-
-    private void onLoaded(Object o, Throwable throwable)
-    {
-        if (throwable != null)
-        {
-            logger.error(String.format("Operation %s failed", this), throwable);
-            fail(throwable);
-        }
-        else
-        {
-            run();
-        }
-    }
-
-    private void state(State state)
-    {
-        this.state = state;
-    }
-
-    private void finish(R result, Throwable failure)
-    {
-        try
-        {
-            if (callback != null)
-                callback.accept(result, failure);
-        }
-        finally
-        {
-            state(failure == null ? FINISHED : FAILED);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    Unseekables<?> keys()
-    {
-        return preLoadContext.keys();
-    }
-
-    private void fail(Throwable throwable)
-    {
-        commandStore.agent().onUncaughtException(throwable);
-        commandStore.checkInStoreThread();
-        Invariants.nonNull(throwable);
-
-        if (state.isComplete())
-            return;
-
-        try
-        {
-            switch (state)
-            {
-                case COMPLETING:
-                    break; // everything's cleaned up, invoke callback
-                case RUNNING:
-                    context.revertChanges();
-                case PREPARING:
-                    commandStore.abortCurrentOperation();
-                case LOADING:
-                    context.releaseResources(commandStore);
-                case INITIALIZED:
-                    break; // nothing to clean up, call callback
-            }
-            if (commandStore.hasSafeStore())
-                commandStore.agent().onUncaughtException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
-        }
-        catch (Throwable cleanup)
-        {
-            commandStore.agent().onUncaughtException(cleanup);
-            throwable.addSuppressed(cleanup);
-        }
-
-        finish(null, throwable);
-    }
-
-    // return true iff ready to run
-    protected boolean runInternal(boolean loadOnly)
-    {
-        switch (state)
-        {
-            default: throw new IllegalStateException("Unexpected state " + state);
-            case INITIALIZED:
-                state(LOADING);
-            case LOADING:
-                if (!loader.load(preLoadContext.primaryTxnId(), context, this::onLoaded))
-                    return false;
-                state(PREPARING);
-                if (loadOnly)
-                    return true;
-            case PREPARING:
-                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.commandsForKey, context.commandsForRanges);
-                state(RUNNING);
-            case RUNNING:
-
-                result = apply(safeStore);
-                // TODO (required): currently, we are not very efficient about ensuring that we persist the absolute minimum amount of state. Improve that.
-                List<SavedCommand.DiffWriter> diffs = null;
-                for (AccordSafeCommand commandState : context.commands.values())
-                {
-                    SavedCommand.DiffWriter diff = commandState.diff();
-                    if (diff == null)
-                        continue;
-                    if (diffs == null)
-                        diffs = new ArrayList<>(context.commands.size());
-                    diffs.add(diff);
-                    if (CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED.getBoolean())
-                    {
-                        if (sanityCheck == null)
-                            sanityCheck = new ArrayList<>(context.commands.size());
-                        sanityCheck.add(commandState.current());
-                    }
-                }
-
-                boolean flushed = false;
-                if (diffs != null || safeStore.fieldUpdates() != null)
-                {
-                    Runnable onFlush = () -> finish(result, null);
-                    if (safeStore.fieldUpdates() != null)
-                        commandStore.persistFieldUpdates(safeStore.fieldUpdates(), diffs == null ? onFlush : null);
-                    if (diffs != null)
-                        appendCommands(diffs, onFlush);
-                    flushed = true;
-                }
-
-                commandStore.completeOperation(safeStore);
-                context.releaseResources(commandStore);
-                state(COMPLETING);
-                if (flushed)
-                    return false;
-
-            case COMPLETING:
-                finish(result, null);
-            case FINISHED:
-            case FAILED:
-                break;
-        }
-
-        return false;
-    }
-
-    private void appendCommands(List<SavedCommand.DiffWriter> diffs, Runnable onFlush)
-    {
-        if (sanityCheck != null)
-        {
-            Invariants.checkState(CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED.getBoolean());
-            Condition condition = Condition.newOneTimeCondition();
-            this.commandStore.appendCommands(diffs, condition::signal);
-            condition.awaitUninterruptibly();
-
-            for (Command check : sanityCheck)
-                this.commandStore.sanityCheckCommand(check);
-
-            if (onFlush != null) onFlush.run();
-        }
-        else
-        {
-            this.commandStore.appendCommands(diffs, onFlush);
-        }
-    }
-
-    @Override
-    public void run()
-    {
-        setLoggingIds();
-        logger.trace("Running {} with state {}", this, state);
-        try
-        {
-            commandStore.checkInStoreThread();
-            commandStore.setCurrentOperation(this);
-            try
-            {
-                runInternal(false);
-            }
-            catch (Throwable t)
-            {
-                logger.error("Operation {} failed", this, t);
-                fail(t);
-            }
-            finally
-            {
-                commandStore.unsetCurrentOperation(this);
-            }
-        }
-        finally
-        {
-            logger.trace("Exiting {}", this);
-            clearLoggingIds();
-        }
-    }
-
-    private boolean preRun()
-    {
-        commandStore.checkInStoreThread();
-        try
-        {
-            return runInternal(true);
-        }
-        catch (Throwable t)
-        {
-            logger.error("Operation {} failed", this, t);
-            fail(t);
-            return false;
-        }
-    }
-
-    @Override
-    public Cancellable start(BiConsumer<? super R, Throwable> callback)
-    {
-        Invariants.checkState(this.callback == null);
-        this.callback = callback;
-        if (!commandStore.inStore() || preRun())
-            commandStore.executor().execute(this);
-        return this;
-    }
-
-    @Override
-    public void cancel()
-    {
-    }
-
-    static class ForFunction<R> extends AsyncOperation<R>
-    {
-        private final Function<? super SafeCommandStore, R> function;
-
-        public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function<? super SafeCommandStore, R> function)
-        {
-            super(commandStore, loadCtx);
-            this.function = function;
-        }
-
-        @Override
-        public R apply(SafeCommandStore commandStore)
-        {
-            return function.apply(commandStore);
-        }
-    }
-
-    public static <T> AsyncOperation<T> create(CommandStore commandStore, PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
-    {
-        return new ForFunction<>((AccordCommandStore) commandStore, loadCtx, function);
-    }
-
-    // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names.
-    static class ForConsumer extends AsyncOperation<Void>
-    {
-        private final Consumer<? super SafeCommandStore> consumer;
-
-        public ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer<? super SafeCommandStore> consumer)
-        {
-            super(commandStore, loadCtx);
-            this.consumer = consumer;
-        }
-
-        @Override
-        public Void apply(SafeCommandStore commandStore)
-        {
-            consumer.accept(commandStore);
-            return null;
-        }
-    }
-
-    public static AsyncOperation<Void> create(CommandStore commandStore, PreLoadContext loadCtx, Consumer<? super SafeCommandStore> consumer)
-    {
-        return new ForConsumer((AccordCommandStore) commandStore, loadCtx, consumer);
-    }
-}

src/java/org/apache/cassandra/service/accord/events/CacheEvents.java

@@ -30,7 +30,7 @@ import jdk.jfr.StackTrace;
 @StackTrace(false)
 public abstract class CacheEvents extends Event
 {
-    public int store;
+    public int shard;
     public String instance;
     public String key;
     public String status;

src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java

@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.service.accord.interop;
 
-import java.util.BitSet;
 import javax.annotation.Nullable;
 
 import accord.api.LocalListeners;
@@ -42,12 +41,12 @@ import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.primitives.Writes;
 import accord.topology.Topologies;
+import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType;
 import org.apache.cassandra.service.accord.serializers.ApplySerializers.ApplySerializer;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
-import org.jctools.queues.MpscChunkedArrayQueue;
 
 import static accord.utils.Invariants.checkState;
 import static com.google.common.base.Preconditions.checkArgument;
@@ -83,9 +82,9 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         }
     };
 
-    transient BitSet waitingOn;
     transient int waitingOnCount;
-    final MpscChunkedArrayQueue<LocalListeners.Registered> listeners = new MpscChunkedArrayQueue<>(4, 1 << 30);
+    transient Int2ObjectHashMap<LocalListeners.Registered> listeners;
+    boolean failed;
 
     private AccordInteropApply(Kind kind, TxnId txnId, Route<?> route, long waitForEpoch, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, @Nullable FullRoute<?> fullRoute, Writes writes, Result result)
     {
@@ -97,14 +96,6 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         super(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
     }
 
-    @Override
-    public void process()
-    {
-        waitingOn = new BitSet();
-        super.process();
-    }
-
-
     @Override
     public ApplyReply apply(SafeCommandStore safeStore, StoreParticipants participants)
     {
@@ -133,12 +124,20 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             case PreCommitted:
             case Committed:
             case PreApplied:
+                LocalListeners.Registered listener = safeStore.register(txnId, this);
                 synchronized (this)
                 {
-                    waitingOn.set(safeStore.commandStore().id());
-                    ++waitingOnCount;
+                    if (!failed)
+                    {
+                        if (listeners == null)
+                            listeners = new Int2ObjectHashMap<>();
+                        listeners.put(safeStore.commandStore().id(), listener);
+                        ++waitingOnCount;
+                        listener = null;
+                    }
                 }
-                listeners.add(safeStore.register(txnId, this));
+                if (listener != null)
+                    listener.cancel();
                 break;
 
             case Applied:
@@ -155,9 +154,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         // and prevents races where we respond before dispatching all the required reads (if the reads are
         // completing faster than the reads can be setup on all required shards)
         if (-1 == --waitingOnCount)
-        {
             node.reply(replyTo, replyContext, ApplyReply.Applied, null);
-        }
     }
 
     @Override
@@ -169,7 +166,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
     }
 
     @Override
-    public void accept(ApplyReply reply, Throwable failure)
+    protected void acceptInternal(ApplyReply reply, Throwable failure)
     {
         if (reply == ApplyReply.Insufficient)
         {
@@ -181,7 +178,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         {
             node.reply(replyTo, replyContext, null, failure);
             node.agent().onUncaughtException(failure);
-            cancel();
+            fail();
         }
 
         // Unless failed always ack to indicate setup has completed otherwise the counter never gets to -1
@@ -189,11 +186,18 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             ack();
     }
 
-    private void cancel()
+    private void fail()
     {
-        listeners.drain(LocalListeners.Registered::cancel);
+        Int2ObjectHashMap<LocalListeners.Registered> listeners;
+        synchronized (this)
+        {
+            failed = true;
+            listeners = this.listeners;
+            this.listeners = null;
+        }
+        listeners.forEach((i, v) -> v.cancel());
     }
-    
+
     @Override
     public TxnId primaryTxnId()
     {
@@ -233,7 +237,6 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
     public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand)
     {
         Command command = safeCommand.current();
-
         switch (command.status())
         {
             default: throw new AssertionError();
@@ -251,7 +254,14 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             case Truncated:
         }
 
-        ack();
+        synchronized (this)
+        {
+            if (failed)
+                return false;
+
+            listeners.remove(safeStore.commandStore().id());
+            ack();
+        }
         return false;
     }
 }

src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java

@@ -143,10 +143,11 @@ public class CommandStoreSerializers
             out.writeUnsignedVInt(t.startOwnershipEpoch);
             if (t.endOwnershipEpoch == Long.MAX_VALUE) out.writeUnsignedVInt(0L);
             else out.writeUnsignedVInt(1 + t.endOwnershipEpoch - t.startOwnershipEpoch);
+            CommandSerializers.txnId.serialize(t.locallyWitnessedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.locallyAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.locallyDecidedAndAppliedOrInvalidatedBefore, out, version);
-            CommandSerializers.txnId.serialize(t.shardAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.shardOnlyAppliedOrInvalidatedBefore, out, version);
+            CommandSerializers.txnId.serialize(t.shardAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.gcBefore, out, version);
             CommandSerializers.txnId.serialize(t.bootstrappedAt, out, version);
             CommandSerializers.nullableTimestamp.serialize(t.staleUntilAtLeast, out, version);
@@ -160,14 +161,15 @@ public class CommandStoreSerializers
             long endEpoch = in.readUnsignedVInt();
             if (endEpoch == 0) endEpoch = Long.MAX_VALUE;
             else endEpoch = endEpoch - 1 + startEpoch;
+            TxnId locallyWitnessedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId locallyAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId locallyDecidedAndAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
-            TxnId shardAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId shardOnlyAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
+            TxnId shardAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId gcBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId bootstrappedAt = CommandSerializers.txnId.deserialize(in, version);
             Timestamp staleUntilAtLeast = CommandSerializers.nullableTimestamp.deserialize(in, version);
-            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyWitnessedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         @Override
@@ -176,10 +178,11 @@ public class CommandStoreSerializers
             long size = TokenRange.serializer.serializedSize((TokenRange) t.range, version);
             size += TypeSizes.sizeofUnsignedVInt(t.startOwnershipEpoch);
             size += TypeSizes.sizeofUnsignedVInt(t.endOwnershipEpoch == Long.MAX_VALUE ? 0 : 1 + t.endOwnershipEpoch - t.startOwnershipEpoch);
+            size += CommandSerializers.txnId.serializedSize(t.locallyWitnessedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.locallyAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.locallyDecidedAndAppliedOrInvalidatedBefore, version);
-            size += CommandSerializers.txnId.serializedSize(t.shardAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.shardOnlyAppliedOrInvalidatedBefore, version);
+            size += CommandSerializers.txnId.serializedSize(t.shardAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.gcBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.bootstrappedAt, version);
             size += CommandSerializers.nullableTimestamp.serializedSize(t.staleUntilAtLeast, version);

src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java

@@ -153,9 +153,9 @@ public class ReadDataSerializers
             CommandSerializers.txnId.serialize(read.txnId, out, version);
             KeySerializers.participants.serialize(read.readScope, out, version);
             out.writeUnsignedVInt(read.executeAtEpoch);
-            CommandSerializers.partialTxn.serialize(read.partialTxn, out, version);
-            DepsSerializer.partialDeps.serialize(read.partialDeps, out, version);
-            KeySerializers.fullRoute.serialize(read.route, out, version);
+            CommandSerializers.partialTxn.serialize(read.partialTxn(), out, version);
+            DepsSerializer.partialDeps.serialize(read.partialDeps(), out, version);
+            KeySerializers.fullRoute.serialize(read.route(), out, version);
         }
 
         @Override
@@ -176,9 +176,9 @@ public class ReadDataSerializers
             return CommandSerializers.txnId.serializedSize(read.txnId, version)
                    + KeySerializers.participants.serializedSize(read.readScope, version)
                    + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
-                   + CommandSerializers.partialTxn.serializedSize(read.partialTxn, version)
-                   + DepsSerializer.partialDeps.serializedSize(read.partialDeps, version)
-                   + KeySerializers.fullRoute.serializedSize(read.route, version);
+                   + CommandSerializers.partialTxn.serializedSize(read.partialTxn(), version)
+                   + DepsSerializer.partialDeps.serializedSize(read.partialDeps(), version)
+                   + KeySerializers.fullRoute.serializedSize(read.route(), version);
         }
     };
 

src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java

@@ -22,8 +22,6 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.SortedSet;
-import java.util.TreeSet;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -63,9 +61,10 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
 
     private Keys extractItemKeys()
     {
-        SortedSet<PartitionKey> keysSet = new TreeSet<>(Key::compareTo);
-        forEach(i -> keysSet.add(getKey(i)));
-        return new Keys(keysSet);
+        Key[] keys = new Key[size()];
+        for (int i = 0 ; i < keys.length ; ++i)
+            keys[i] = getKey(items[i]);
+        return Keys.ofSorted(keys);
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java

@@ -71,9 +71,15 @@ public abstract class AbstractSerialized<T>
 
     protected T get()
     {
-        if (memoized == null)
-            memoized = AccordSerializers.deserialize(bytes, serializer());
-        return memoized;
+        T result = memoized;
+        if (result == null)
+            memoized = result = AccordSerializers.deserialize(bytes, serializer());
+        return result;
+    }
+
+    public void unmemoize()
+    {
+        memoized = null;
     }
 
     protected ByteBuffer bytes()

src/java/org/apache/cassandra/service/accord/txn/TxnRead.java

@@ -148,6 +148,12 @@ public class TxnRead extends AbstractKeySorted<TxnNamedRead> implements Read
         return new TxnNamedRead[size];
     }
 
+    public void unmemoize()
+    {
+        for (int i = 0 ; i < size() ; ++i)
+            items[i].unmemoize();
+    }
+
     @Override
     public Keys keys()
     {

src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java

@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
 import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.Write;
-import accord.impl.AbstractSafeCommandStore;
 import accord.impl.TimestampsForKey;
 import accord.impl.TimestampsForKeys;
 import accord.local.SafeCommandStore;
@@ -185,7 +184,6 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         };
     }
 
-
     /**
      * Partition update that can later be supplemented with data from the read phase
      */
@@ -373,13 +371,19 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         return new Update[size];
     }
 
+    public void unmemoize()
+    {
+        for (int i = 0 ; i < size() ; ++i)
+            items[i].unmemoize();
+    }
+
     @Override
     public AsyncChain<Void> apply(Seekable key, SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, DataStore store, PartialTxn txn)
     {
         // TODO (expected, efficiency): 99.9999% of the time we can just use executeAt.hlc(), so can avoid bringing
         //  cfk into memory by retaining at all times in memory key ranges that are dirty and must use this logic;
         //  any that aren't can just use executeAt.hlc
-        TimestampsForKey cfk = TimestampsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?,?,?>) safeStore, ((Key) key).toUnseekable(), txnId, executeAt, true);
+        TimestampsForKey cfk = TimestampsForKeys.updateLastExecutionTimestamps(safeStore, ((Key) key).toUnseekable(), txnId, executeAt, true);
         long timestamp = AccordSafeTimestampsForKey.timestampMicrosFor(cfk, executeAt, true);
         // TODO (low priority - do we need to compute nowInSeconds, or can we just use executeAt?)
         int nowInSeconds = AccordSafeTimestampsForKey.nowInSecondsFor(cfk, executeAt, true);

src/java/org/apache/cassandra/utils/ByteBufferUtil.java

@@ -768,7 +768,7 @@ public class ByteBufferUtil
 
     public static ByteBuffer bytes(TimeUUID uuid)
     {
-        return bytes(uuid.asUUID());
+        return ByteBuffer.wrap(UUIDGen.decompose(uuid.msb(), uuid.lsb()));
     }
 
     // Returns whether {@code prefix} is a prefix of {@code value}.

src/java/org/apache/cassandra/utils/UUIDGen.java

@@ -45,13 +45,16 @@ public class UUIDGen
     /** decomposes a uuid into raw bytes. */
     public static byte[] decompose(UUID uuid)
     {
-        long most = uuid.getMostSignificantBits();
-        long least = uuid.getLeastSignificantBits();
+        return decompose(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
+    }
+
+    public static byte[] decompose(long msb, long lsb)
+    {
         byte[] b = new byte[16];
         for (int i = 0; i < 8; i++)
         {
-            b[i] = (byte)(most >>> ((7-i) * 8));
-            b[8+i] = (byte)(least >>> ((7-i) * 8));
+            b[i] = (byte)(msb >>> ((7-i) * 8));
+            b[8+i] = (byte)(lsb >>> ((7-i) * 8));
         }
         return b;
     }

src/java/org/apache/cassandra/utils/btree/AbstractBTreeMap.java

@@ -33,11 +33,13 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
 {
     protected final Object[] tree;
     protected final KeyComparator<K, V> comparator;
+    protected final AsymmetricKeyComparator<K> asymmetricComparator;
 
-    protected AbstractBTreeMap(Object[] tree, KeyComparator<K, V> comparator)
+    protected AbstractBTreeMap(Object[] tree, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricComparator)
     {
         this.tree = tree;
         this.comparator = comparator;
+        this.asymmetricComparator = asymmetricComparator;
     }
 
     /**
@@ -93,7 +95,7 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
     {
         if (key == null)
             throw new NullPointerException();
-        Entry<K, V> entry = BTree.find(tree, comparator, new Entry<>((K)key, null));
+        Entry<K, V> entry = (Entry<K, V>) BTree.find(tree, asymmetricComparator, key);
         if (entry != null)
             return entry.getValue();
         return null;
@@ -161,6 +163,22 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
         }
     }
 
+    protected static class AsymmetricKeyComparator<K> implements Comparator<Object>
+    {
+        protected final Comparator<K> keyComparator;
+
+        protected AsymmetricKeyComparator(Comparator<K> keyComparator)
+        {
+            this.keyComparator = keyComparator;
+        }
+
+        @Override
+        public int compare(Object o1, Object o2)
+        {
+            return keyComparator.compare(((Map.Entry<K, ?>)o1).getKey(), (K)o2);
+        }
+    }
+
     static class Entry<K, V> extends AbstractMap.SimpleEntry<K, V>
     {
         public Entry(K key, V value)

src/java/org/apache/cassandra/utils/btree/BTreeBiMap.java

@@ -29,16 +29,32 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 {
     private final Object[] inverse;
     private final KeyComparator<V, K> valueComparator;
+    private final AsymmetricKeyComparator<V> asymmetricValueComparator;
 
     protected static <K, V> BTreeBiMap<K, V> withComparators(Object[] tree, Object [] inverse, Comparator<K> comparator, Comparator<V> valueComparator)
     {
-        return new BTreeBiMap<>(tree, inverse, new KeyComparator<>(comparator), new KeyComparator<>(valueComparator));
+        KeyComparator<K, V> keyComparator = new KeyComparator<>(comparator);
+        AsymmetricKeyComparator<K> asymmetricKeyComparator = new AsymmetricKeyComparator<>(comparator);
+        KeyComparator<V, K> valueKeyComparator;
+        AsymmetricKeyComparator<V> asymmetricValueComparator;
+        if (comparator == valueComparator)
+        {
+            valueKeyComparator = (KeyComparator<V, K>) keyComparator;
+            asymmetricValueComparator = (AsymmetricKeyComparator<V>) asymmetricKeyComparator;
+        }
+        else
+        {
+            valueKeyComparator = new KeyComparator<>(valueComparator);
+            asymmetricValueComparator = new AsymmetricKeyComparator<>(valueComparator);
+        }
+        return new BTreeBiMap<>(tree, inverse, keyComparator, asymmetricKeyComparator, valueKeyComparator, asymmetricValueComparator);
     }
 
-    private BTreeBiMap(Object[] tree, Object [] inverse, KeyComparator<K, V> comparator, KeyComparator<V, K> valueComparator)
+    private BTreeBiMap(Object[] tree, Object [] inverse, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricKeyComparator, KeyComparator<V, K> valueComparator, AsymmetricKeyComparator<V> asymmetricValueComparator)
     {
-        super(tree, comparator);
+        super(tree, comparator, asymmetricKeyComparator);
         this.valueComparator = valueComparator;
+        this.asymmetricValueComparator = asymmetricValueComparator;
         this.inverse = inverse;
     }
 
@@ -55,7 +71,7 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
     @Override
     public BiMap<V, K> inverse()
     {
-        return new BTreeBiMap<>(inverse, tree, valueComparator, comparator);
+        return new BTreeBiMap<>(inverse, tree, valueComparator, asymmetricValueComparator, comparator, asymmetricComparator);
     }
 
     @Override
@@ -72,8 +88,8 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 
         return new BTreeBiMap<>(BTree.update(tree, new Object[]{ entry }, comparator, UpdateFunction.noOp()),
                                 BTree.update(inverse, new Object[] { new AbstractBTreeMap.Entry<>(value, key) }, valueComparator, UpdateFunction.noOp()),
-                                comparator,
-                                valueComparator);
+                                comparator, asymmetricComparator,
+                                valueComparator, asymmetricValueComparator);
     }
 
     @Override
@@ -92,7 +108,7 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 
         Object[] newTree = BTreeRemoval.remove(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
         Object[] newInverse = BTreeRemoval.remove(inverse, valueComparator, new AbstractBTreeMap.Entry<>(existingEntry.getValue(), null));
-        return new BTreeBiMap<>(newTree, newInverse, comparator, valueComparator);
+        return new BTreeBiMap<>(newTree, newInverse, comparator, asymmetricComparator, valueComparator, asymmetricValueComparator);
     }
 
     public Set<V> values()

src/java/org/apache/cassandra/utils/btree/BTreeMap.java

@@ -33,12 +33,12 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
 {
     protected static <K, V> BTreeMap<K, V> withComparator(Object[] tree, Comparator<K> comparator)
     {
-        return new BTreeMap<>(tree, new KeyComparator<>(comparator));
+        return new BTreeMap<>(tree, new KeyComparator<>(comparator), new AsymmetricKeyComparator<>(comparator));
     }
 
-    protected BTreeMap(Object[] tree, KeyComparator<K, V> comparator)
+    protected BTreeMap(Object[] tree, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricComparator)
     {
-        super(tree, comparator);
+        super(tree, comparator, asymmetricComparator);
     }
 
     public static <K, V> BTreeMap<K, V> empty(Comparator<K> comparator)
@@ -61,7 +61,7 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         AbstractBTreeMap.Entry<K, V> existing;
         if ((existing = BTree.find(tree, comparator, entry)) != null && !existing.equals(entry))
             throw new IllegalStateException("Map already contains " + key);
-        return new BTreeMap<>(BTree.update(tree, new Object[]{ entry }, comparator, UpdateFunction.noOp()), comparator);
+        return new BTreeMap<>(BTree.update(tree, BTree.singleton(entry), comparator, UpdateFunction.noOp()), comparator, asymmetricComparator);
     }
 
     public BTreeMap<K, V> withForce(K key, V value)
@@ -69,7 +69,7 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         if (key == null || value == null)
             throw new NullPointerException();
         AbstractBTreeMap.Entry<K, V> entry = new AbstractBTreeMap.Entry<>(key, value);
-        return new BTreeMap<>(BTree.update(tree, new Object[] { entry }, comparator, UpdateFunction.Simple.of((a, b) -> b)), comparator);
+        return new BTreeMap<>(BTree.update(tree, BTree.singleton(entry), comparator, UpdateFunction.Simple.of((a, b) -> b)), comparator, asymmetricComparator);
     }
 
     public BTreeMap<K, V> without(K key)
@@ -77,13 +77,14 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         if (key == null)
             throw new NullPointerException();
 
-        return new BTreeMap<>(BTreeRemoval.remove(tree, comparator, new AbstractBTreeMap.Entry<>(key, null)), comparator);
+        return new BTreeMap<>(BTreeRemoval.remove(tree, asymmetricComparator, key), comparator, asymmetricComparator);
     }
 
     @Override
     public Map.Entry<K, V> lowerEntry(K key)
     {
-        return BTree.lower(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.lower(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -96,7 +97,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> floorEntry(K key)
     {
-        return BTree.floor(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.floor(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -109,7 +111,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> ceilingEntry(K key)
     {
-        return BTree.ceil(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.ceil(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -122,7 +125,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> higherEntry(K key)
     {
-        return BTree.higher(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.higher(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -151,8 +155,9 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public NavigableMap<K, V> descendingMap()
     {
+        Comparator<K> reversed = comparator.keyComparator.reversed();
         return new BTreeMap<>(BTree.build(BulkIterator.of(BTree.iterable(tree, BTree.Dir.DESC).iterator()), BTree.size(tree), UpdateFunction.noOp),
-                              new KeyComparator<>(comparator.keyComparator.reversed()));
+                              new KeyComparator<>(reversed), new AsymmetricKeyComparator<>(reversed));
     }
 
     @Override

src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public class ConcurrentLinkedStack<T>
+{
+    static final class Node<T> extends IntrusiveStack<Node<T>>
+    {
+        final T value;
+        Node(T value)
+        {
+            this.value = value;
+        }
+    }
+
+    private volatile Node<T> head;
+    private static final AtomicReferenceFieldUpdater<ConcurrentLinkedStack, Node> headUpdater = AtomicReferenceFieldUpdater.newUpdater(ConcurrentLinkedStack.class, Node.class, "head");
+
+    public void push(T value)
+    {
+        IntrusiveStack.push(headUpdater, this, (Node)new Node<>(value));
+    }
+
+    public boolean isEmpty()
+    {
+        return head == null;
+    }
+
+    public void drain(Consumer<T> forEach, boolean reverse)
+    {
+        if (isEmpty())
+            return;
+
+        Node<T> head = headUpdater.getAndSet(this, null);
+        if (reverse) head = IntrusiveStack.reverse(head);
+        IntrusiveStack.forEach(head, n -> n.value, forEach);
+    }
+
+    public <P> void drain(BiConsumer<P, T> forEach, P param, boolean reverse)
+    {
+        if (isEmpty())
+            return;
+
+        Node<T> head = headUpdater.getAndSet(this, null);
+        if (reverse) head = IntrusiveStack.reverse(head);
+        IntrusiveStack.forEach(head, n -> n.value, forEach, param);
+    }
+}

src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent;
 
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -189,10 +190,25 @@ public class IntrusiveStack<T extends IntrusiveStack<T>> implements Iterable<T>
     }
 
     protected static <T extends IntrusiveStack<T>> void forEach(T list, Consumer<? super T> forEach)
+    {
+        forEach(list, Function.identity(), forEach);
+    }
+
+    protected static <T extends IntrusiveStack<T>, P> void forEach(T list, BiConsumer<P, ? super T> forEach, P param)
+    {
+        forEach(list, Function.identity(), forEach, param);
+    }
+
+    protected static <T extends IntrusiveStack<T>, V> void forEach(T list, Function<? super T, ? extends V> getter, Consumer<? super V> forEach)
+    {
+        forEach(list, getter, Consumer::accept, forEach);
+    }
+
+    protected static <P, T extends IntrusiveStack<T>, V> void forEach(T list, Function<? super T, ? extends V> getter, BiConsumer<? super P, ? super V> forEach, P param)
     {
         while (list != null)
         {
-            forEach.accept(list);
+            forEach.accept(param, getter.apply(list));
             list = list.next;
         }
     }