diff --git a/build.xml b/build.xml
index e1905a9304..3fe92490d4 100644
--- a/build.xml
+++ b/build.xml
@@ -1040,6 +1040,32 @@
       </checksum>
     </target>
 
+    <target name="bintar" depends="_artifacts-init,check"
+            description="Create Cassandra tarball and maven artifacts">
+      <tar compression="gzip" longfile="gnu"
+        destfile="${build.dir}/${final.name}-bin.tar.gz">
+
+        <!-- Everything but bin/ (default mode) -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}">
+          <include name="**"/>
+          <exclude name="bin/*" />
+          <exclude name="tools/bin/*"/>
+        </tarfileset>
+        <!-- Shell includes in bin/ (default mode) -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}">
+          <include name="bin/*.in.sh" />
+          <include name="tools/bin/*.in.sh" />
+        </tarfileset>
+        <!-- Executable scripts in bin/ -->
+        <tarfileset dir="${dist.dir}" prefix="${final.name}" mode="755">
+          <include name="bin/*"/>
+          <include name="tools/bin/*"/>
+          <exclude name="bin/*.in.sh" />
+          <exclude name="tools/bin/*.in.sh" />
+        </tarfileset>
+      </tar>
+    </target>
+
   <!-- Wrapper of build-test without dependencies, so both that target and its dependencies are skipped if the property
     no-build-test is true. This is meant to be used to run tests without actually building them, provided that they have
     been built before. All test targets depend on this, so one can run them using the no-build-test property.
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 8832243f6e..fe80fda164 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -2657,7 +2657,10 @@ storage_compatibility_mode: NONE
 #  journal_directory:
 #
 #  # The number of Accord shards on this node; -1 means use the number of cores
-#  shard_count: -1
+#  queue_shard_count: -1
+#
+#  # The number of Accord shards on this node; -1 means use the number of cores
+#  command_store_shard_count: -1
 #
 #  # Recover delay: the time between a transaction being initiated and a remote replica being willing to interrupt it to complete it
 #  recover_delay: 1s
diff --git a/modules/accord b/modules/accord
index 4ec8d262a7..8bd9d69803 160000
--- a/modules/accord
+++ b/modules/accord
@@ -1 +1 @@
-Subproject commit 4ec8d262a750a76744b7f6991b711f85fa41a89a
+Subproject commit 8bd9d6980350fa68a1db676a7b10940cf0541fb5
diff --git a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
index b576551ac0..c9487e4164 100644
--- a/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
+++ b/src/java/org/apache/cassandra/concurrent/InfiniteLoopExecutor.java
@@ -108,7 +108,6 @@ public class InfiniteLoopExecutor implements Interruptible
         };
     }
 
-
     private void loop()
     {
         boolean interrupted = false;
@@ -194,6 +193,11 @@ public class InfiniteLoopExecutor implements Interruptible
         return isTerminated();
     }
 
+    public long threadId()
+    {
+        return thread.getId();
+    }
+
     @VisibleForTesting
     public boolean isAlive()
     {
diff --git a/src/java/org/apache/cassandra/concurrent/Stage.java b/src/java/org/apache/cassandra/concurrent/Stage.java
index 910ac40a5b..b4f0540d93 100644
--- a/src/java/org/apache/cassandra/concurrent/Stage.java
+++ b/src/java/org/apache/cassandra/concurrent/Stage.java
@@ -47,7 +47,7 @@ public enum Stage
     MUTATION           (true,  "MutationStage",         "request",  DatabaseDescriptor::getConcurrentWriters,        DatabaseDescriptor::setConcurrentWriters,        Stage::multiThreadedLowSignalStage),
     COUNTER_MUTATION   (true,  "CounterMutationStage",  "request",  DatabaseDescriptor::getConcurrentCounterWriters, DatabaseDescriptor::setConcurrentCounterWriters, Stage::multiThreadedLowSignalStage),
     VIEW_MUTATION      (true,  "ViewMutationStage",     "request",  DatabaseDescriptor::getConcurrentViewWriters,    DatabaseDescriptor::setConcurrentViewWriters,    Stage::multiThreadedLowSignalStage),
-    ACCORD_MIGRATION   (false, "AccordMigrationStage",  "request",  DatabaseDescriptor::getConcurrentAccordOps,      DatabaseDescriptor::setConcurrentAccordOps,      Stage::multiThreadedLowSignalStage),
+    ACCORD_MIGRATION   (false, "AccordMigrationStage", "request", DatabaseDescriptor::getAccordConcurrentOps, DatabaseDescriptor::setConcurrentAccordOps, Stage::multiThreadedLowSignalStage),
     GOSSIP             (true,  "GossipStage",           "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
     REQUEST_RESPONSE   (false, "RequestResponseStage",  "request",  FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedLowSignalStage),
     ANTI_ENTROPY       (false, "AntiEntropyStage",      "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
@@ -59,7 +59,6 @@ public enum Stage
     PAXOS_REPAIR       (false, "PaxosRepairStage",      "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     INTERNAL_METADATA  (false, "InternalMetadataStage", "internal", FBUtilities::getAvailableProcessors,             null,                                            Stage::multiThreadedStage),
     FETCH_LOG          (false, "MetadataFetchLogStage", "internal", () -> 1,                                         null,                                            Stage::singleThreadedStage),
-    ACCORD_RANGE_LOADER(false, "AccordRangeLoader",     "internal", () -> 4,                                         null,                                            Stage::multiThreadedStage),
     ;
     public final String jmxName;
     private final Supplier<ExecutorPlus> executorSupplier;
diff --git a/src/java/org/apache/cassandra/config/AccordSpec.java b/src/java/org/apache/cassandra/config/AccordSpec.java
index 74504326ab..b445e4492e 100644
--- a/src/java/org/apache/cassandra/config/AccordSpec.java
+++ b/src/java/org/apache/cassandra/config/AccordSpec.java
@@ -21,11 +21,14 @@ package org.apache.cassandra.config;
 import java.util.concurrent.TimeUnit;
 
 import accord.primitives.TxnId;
+import accord.utils.Invariants;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 
 import static accord.primitives.Routable.Domain.Range;
+import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_POOL_PER_SHARD;
+import static org.apache.cassandra.config.AccordSpec.QueueSubmissionModel.SYNC;
 
 public class AccordSpec
 {
@@ -35,18 +38,99 @@ public class AccordSpec
 
     public volatile boolean enable_journal_compaction = true;
 
-    public volatile OptionaldPositiveInt shard_count = OptionaldPositiveInt.UNDEFINED;
+    public enum QueueShardModel
+    {
+        /**
+         * Same number of threads as queue shards, but the shard lock is held only while managing the queue,
+         * so that submitting threads may queue load/save work.
+         *
+         * The global READ and WRITE stages are used for IO.
+         */
+        THREAD_PER_SHARD,
+
+        /**
+         * Same number of threads as shards, and the shard lock is held for the duration of serving requests.
+         * The global READ and WRITE stages are used for IO.
+         */
+        THREAD_PER_SHARD_SYNC_QUEUE,
+
+        /**
+         * More threads than shards. Threads update transaction state as well as performing IO, minimising context switching.
+         * Fewer shards is generally better, until queue-contention is encountered.
+         */
+        THREAD_POOL_PER_SHARD,
+
+        /**
+         * More threads than shards. Threads update transaction state only, relying on READ and WRITE stages for IO.
+         * Fewer shards is generally better, until queue-contention is encountered.
+         */
+        THREAD_POOL_PER_SHARD_EXCLUDES_IO,
+    }
+
+    public enum QueueSubmissionModel
+    {
+        /**
+         * The queue workers and all submissions require ownership of the lock.
+         */
+        SYNC,
+
+        /**
+         * The queue workers and some submissions require ownership of the lock.
+         * That is, if the lock is available on submission we take it; if it is not we try to guarantee that
+         * another thread will witness the work submission promptly, but if we cannot we wait for the lock
+         * to ensure work is scheduled.
+         */
+        SEMI_SYNC,
+
+        /**
+         * The queue workers only require ownership of the lock, submissions happens fully asynchronously.
+         */
+        ASYNC,
+
+        /**
+         * The queue is backed by submission to a single-threaded plain executor.
+         * This implementation does not honur the sharding model option.
+         *
+         * Note: this isn't intended to be used by real clusters.
+         */
+        EXEC_ST
+    }
+
+    public QueueShardModel queue_shard_model = THREAD_POOL_PER_SHARD;
+    public QueueSubmissionModel queue_submission_model = SYNC;
+
+    /**
+     * The number of queue (and cache) shards.
+     */
+    public volatile OptionaldPositiveInt queue_shard_count = OptionaldPositiveInt.UNDEFINED;
+
+    /**
+     * The target number of command stores to create per topology shard.
+     * This determines the amount of execution parallelism possible for a given table/shard on the host.
+     * More shards means more parallelism, but more state.
+     *
+     * TODO (expected): make this a table property
+     * TODO (expected): adjust this by proportion of ring
+     */
+    public volatile OptionaldPositiveInt command_store_shard_count = OptionaldPositiveInt.UNDEFINED;
+
+    public volatile OptionaldPositiveInt max_queued_loads = OptionaldPositiveInt.UNDEFINED;
+    public volatile OptionaldPositiveInt max_queued_range_loads = OptionaldPositiveInt.UNDEFINED;
+
+    public DataStorageSpec.LongMebibytesBound cache_size = null;
+    public DataStorageSpec.LongMebibytesBound working_set_size = null;
+    public boolean shrink_cache_entries_before_eviction = true;
 
     // TODO (expected): we should be able to support lower recover delays, at least for txns
     public volatile DurationSpec.IntMillisecondsBound recover_delay = new DurationSpec.IntMillisecondsBound(5000);
-    public volatile DurationSpec.IntMillisecondsBound range_sync_recover_delay = new DurationSpec.IntMillisecondsBound("5m");
+    public volatile DurationSpec.IntMillisecondsBound range_syncpoint_recover_delay = new DurationSpec.IntMillisecondsBound("5m");
     public String slowPreAccept = "30ms <= p50*2 <= 100ms";
     public String slowRead = "30ms <= p50*2 <= 100ms";
 
     public long recoveryDelayFor(TxnId txnId, TimeUnit unit)
     {
         if (txnId.isSyncPoint() && txnId.is(Range))
-            return range_sync_recover_delay.to(unit);
+            return range_syncpoint_recover_delay.to(unit);
         return recover_delay.to(unit);
     }
 
@@ -63,7 +147,7 @@ public class AccordSpec
 
     public DurationSpec.IntMillisecondsBound barrier_max_backoff = new DurationSpec.IntMillisecondsBound("10m");
 
-    public DurationSpec.IntMillisecondsBound range_barrier_timeout = new DurationSpec.IntMillisecondsBound("2m");
+    public DurationSpec.IntMillisecondsBound range_syncpoint_timeout = new DurationSpec.IntMillisecondsBound("2m");
 
     public volatile DurationSpec.IntSecondsBound fast_path_update_delay = new DurationSpec.IntSecondsBound("60m");
 
@@ -92,7 +176,7 @@ public class AccordSpec
      * default transactional mode for tables created by this node when no transactional mode has been specified in the DDL
      */
     public TransactionalMode default_transactional_mode = TransactionalMode.off;
-    public boolean ephemeralReadEnabled = false;
+    public boolean ephemeralReadEnabled = true;
     public boolean state_cache_listener_jfr_enabled = true;
     public final JournalSpec journal = new JournalSpec();
     public final MinEpochRetrySpec minEpochSyncRetry = new MinEpochRetrySpec();
@@ -110,9 +194,22 @@ public class AccordSpec
         public int segmentSize = 32 << 20;
         public FailurePolicy failurePolicy = FailurePolicy.STOP;
         public FlushMode flushMode = FlushMode.PERIODIC;
-        public DurationSpec.IntMillisecondsBound flushPeriod; // pulls default from 'commitlog_sync_period'
-        public DurationSpec.IntMillisecondsBound periodicFlushLagBlock = new DurationSpec.IntMillisecondsBound("1500ms");
+        public volatile DurationSpec flushPeriod; // pulls default from 'commitlog_sync_period'
+        public DurationSpec periodicFlushLagBlock = new DurationSpec.IntMillisecondsBound("1500ms");
         public DurationSpec.IntMillisecondsBound compactionPeriod = new DurationSpec.IntMillisecondsBound("60000ms");
+        private volatile long flushCombinedBlockPeriod = Long.MIN_VALUE;
+
+        public void setFlushPeriod(DurationSpec newFlushPeriod)
+        {
+            flushPeriod = newFlushPeriod;
+            flushCombinedBlockPeriod = Long.MIN_VALUE;
+        }
+
+        public void setPeriodicFlushLagBlock(DurationSpec newPeriodicFlushLagBlock)
+        {
+            periodicFlushLagBlock = newPeriodicFlushLagBlock;
+            flushCombinedBlockPeriod = Long.MIN_VALUE;
+        }
 
         @Override
         public int segmentSize()
@@ -139,24 +236,32 @@ public class AccordSpec
         }
 
         @Override
-        public int compactionPeriodMillis()
+        public long compactionPeriod(TimeUnit unit)
         {
-            return compactionPeriod.toMilliseconds();
+            return compactionPeriod.to(unit);
         }
 
         @JsonIgnore
         @Override
-        public int flushPeriodMillis()
+        public long flushPeriod(TimeUnit units)
         {
-            return flushPeriod == null ? DatabaseDescriptor.getCommitLogSyncPeriod()
-                                       : flushPeriod.toMilliseconds();
+            return flushPeriod.to(units);
         }
 
         @JsonIgnore
         @Override
-        public int periodicFlushLagBlock()
+        public long periodicBlockPeriod(TimeUnit units)
         {
-            return periodicFlushLagBlock.toMilliseconds();
+            long nanos = flushCombinedBlockPeriod;
+            if (nanos >= 0)
+                return units.convert(nanos, TimeUnit.NANOSECONDS);
+
+            long flushPeriodNanos = flushPeriod(TimeUnit.NANOSECONDS);
+            Invariants.checkState(flushPeriodNanos > 0);
+            nanos = periodicFlushLagBlock.to(TimeUnit.NANOSECONDS) + flushPeriodNanos;
+            // it is possible for this to race and cache the wrong value after an update
+            flushCombinedBlockPeriod = nanos;
+            return nanos;
         }
 
         /**
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index c33c9bef24..615480518d 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -501,7 +501,6 @@ public class Config
     public volatile int counter_cache_keys_to_save = Integer.MAX_VALUE;
 
     public DataStorageSpec.LongMebibytesBound paxos_cache_size = null;
-    public DataStorageSpec.LongMebibytesBound accord_cache_size = null;
 
     public DataStorageSpec.LongMebibytesBound consensus_migration_cache_size = null;
 
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 4f6cebbd52..25035cb610 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -164,6 +164,7 @@ import static org.apache.cassandra.db.ConsistencyLevel.ONE;
 import static org.apache.cassandra.db.ConsistencyLevel.QUORUM;
 import static org.apache.cassandra.io.util.FileUtils.ONE_GIB;
 import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
+import static org.apache.cassandra.journal.Params.FlushMode.PERIODIC;
 import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome;
 
 public class DatabaseDescriptor
@@ -226,6 +227,7 @@ public class DatabaseDescriptor
     private static long keyCacheSizeInMiB;
     private static long paxosCacheSizeInMiB;
     private static long accordCacheSizeInMiB;
+    private static long accordWorkingSetSizeInMiB;
     private static long consensusMigrationCacheSizeInMiB;
     private static long counterCacheSizeInMiB;
     private static long indexSummaryCapacityInMiB;
@@ -611,6 +613,16 @@ public class DatabaseDescriptor
             logger.debug("Syncing log with a period of {}", conf.commitlog_sync_period.toString());
         }
 
+        if (conf.accord.journal.flushPeriod == null)
+        {
+            conf.accord.journal.flushPeriod = conf.commitlog_sync_period;
+            if (conf.accord.journal.flushMode == PERIODIC && conf.commitlog_sync_period.toMilliseconds() == 0)
+            {
+                logger.warn("Accord journal is configured in periodic mode, while Cassandra commit log is configured in {} mode", conf.commitlog_sync);
+                conf.accord.journal.flushPeriod = conf.accord.journal.periodicFlushLagBlock;
+            }
+        }
+
         /* evaluate the DiskAccessMode Config directive, which also affects indexAccessMode selection */
         if (conf.disk_access_mode == DiskAccessMode.auto || conf.disk_access_mode == DiskAccessMode.mmap_index_only)
         {
@@ -964,18 +976,32 @@ public class DatabaseDescriptor
 
         try
         {
-            // if paxosCacheSizeInMiB option was set to "auto" then size of the cache should be "max(10% of Heap (in MB), 1MB)
-            accordCacheSizeInMiB = (conf.accord_cache_size == null)
+            // if accordCacheSizeInMiB option was set to "auto" then size of the cache should be "max(10% of Heap (in MB), 1MB)
+            accordCacheSizeInMiB = (conf.accord.cache_size == null)
                                   ? Math.max(1, (int) ((Runtime.getRuntime().totalMemory() * 0.10) / 1024 / 1024))
-                                  : conf.accord_cache_size.toMebibytes();
+                                  : conf.accord.cache_size.toMebibytes();
 
             if (accordCacheSizeInMiB < 0)
                 throw new NumberFormatException(); // to escape duplicating error message
         }
         catch (NumberFormatException e)
         {
-            throw new ConfigurationException("paxos_cache_size option was set incorrectly to '"
-                                             + conf.paxos_cache_size + "', supported values are <integer> >= 0.", false);
+            throw new ConfigurationException("accord.cache_size option was set incorrectly to '"
+                                             + conf.accord.cache_size + "', supported values are <integer> >= 0.", false);
+        }
+
+        try
+        {
+            // if accordWorkingSetSizeInMiB option was set to "auto" then size of the working set should be "max(5% of Heap (in MB), 1MB)
+            // if negative, there is no limit
+            accordWorkingSetSizeInMiB = (conf.accord.working_set_size == null)
+                                  ? Math.max(1, (int) ((Runtime.getRuntime().totalMemory() * 0.05) / 1024 / 1024))
+                                  : conf.accord.working_set_size.toMebibytes();
+        }
+        catch (NumberFormatException e)
+        {
+            throw new ConfigurationException("accord.working_set_size option was set incorrectly to '"
+                                             + conf.accord.working_set_size + "', supported values are <integer> >= 0.", false);
         }
 
         try
@@ -2729,7 +2755,7 @@ public class DatabaseDescriptor
         conf.concurrent_materialized_view_writes = concurrent_materialized_view_writes;
     }
 
-    public static int getConcurrentAccordOps()
+    public static int getAccordConcurrentOps()
     {
         return conf.concurrent_accord_operations;
     }
@@ -3659,41 +3685,6 @@ public class DatabaseDescriptor
         return conf.paxos_topology_repair_strict_each_quorum;
     }
 
-    public static AccordSpec getAccord()
-    {
-        return conf == null ? null : conf.accord;
-    }
-
-    public static AccordSpec.TransactionalRangeMigration getTransactionalRangeMigration()
-    {
-        return conf.accord.range_migration;
-    }
-
-    public static void setTransactionalRangeMigration(AccordSpec.TransactionalRangeMigration val)
-    {
-        conf.accord.range_migration = Preconditions.checkNotNull(val);
-    }
-
-    public static int getAccordBarrierRetryAttempts()
-    {
-        return conf.accord.barrier_retry_attempts;
-    }
-
-    public static long getAccordBarrierRetryInitialBackoffMillis()
-    {
-        return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds();
-    }
-
-    public static long getAccordBarrierRetryMaxBackoffMillis()
-    {
-        return conf.accord.barrier_max_backoff.toMilliseconds();
-    }
-
-    public static long getAccordRangeBarrierTimeoutNanos()
-    {
-        return conf.accord.range_barrier_timeout.to(TimeUnit.NANOSECONDS);
-    }
-
     public static TransactionalMode defaultTransactionalMode()
     {
         return conf.accord.default_transactional_mode;
@@ -4296,6 +4287,11 @@ public class DatabaseDescriptor
         return accordCacheSizeInMiB;
     }
 
+    public static long getAccordWorkingSetSizeInMiB()
+    {
+        return accordWorkingSetSizeInMiB;
+    }
+
     public static long getConsensusMigrationCacheSizeInMiB()
     {
         return consensusMigrationCacheSizeInMiB;
@@ -5313,6 +5309,42 @@ public class DatabaseDescriptor
         }
     }
 
+
+    public static AccordSpec getAccord()
+    {
+        return conf.accord;
+    }
+
+    public static AccordSpec.TransactionalRangeMigration getTransactionalRangeMigration()
+    {
+        return conf.accord.range_migration;
+    }
+
+    public static void setTransactionalRangeMigration(AccordSpec.TransactionalRangeMigration val)
+    {
+        conf.accord.range_migration = Preconditions.checkNotNull(val);
+    }
+
+    public static int getAccordBarrierRetryAttempts()
+    {
+        return conf.accord.barrier_retry_attempts;
+    }
+
+    public static long getAccordBarrierRetryInitialBackoffMillis()
+    {
+        return conf.accord.barrier_retry_inital_backoff_millis.toMilliseconds();
+    }
+
+    public static long getAccordBarrierRetryMaxBackoffMillis()
+    {
+        return conf.accord.barrier_max_backoff.toMilliseconds();
+    }
+
+    public static long getAccordRangeSyncPointTimeoutNanos()
+    {
+        return conf.accord.range_syncpoint_timeout.to(TimeUnit.NANOSECONDS);
+    }
+
     public static boolean getAccordTransactionsEnabled()
     {
         return conf.accord.enabled;
@@ -5323,9 +5355,69 @@ public class DatabaseDescriptor
         conf.accord.enabled = b;
     }
 
-    public static int getAccordShardCount()
+    public static AccordSpec.QueueShardModel getAccordQueueShardModel()
     {
-        return conf.accord.shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+        return conf.accord.queue_shard_model;
+    }
+
+    public static AccordSpec.QueueSubmissionModel getAccordQueueSubmissionModel()
+    {
+        return conf.accord.queue_submission_model;
+    }
+
+    public static int getAccordQueueShardCount()
+    {
+        switch (getAccordQueueShardModel())
+        {
+            default: throw new AssertionError("Unhandled queue_shard_model: " + conf.accord.queue_shard_model);
+            case THREAD_PER_SHARD:
+            case THREAD_PER_SHARD_SYNC_QUEUE:
+                return conf.accord.queue_shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+            case THREAD_POOL_PER_SHARD:
+            case THREAD_POOL_PER_SHARD_EXCLUDES_IO:
+                int defaultMax = getAccordQueueSubmissionModel() == AccordSpec.QueueSubmissionModel.SYNC ? 8 : 4;
+                return conf.accord.queue_shard_count.or(Math.min(defaultMax, DatabaseDescriptor.getAvailableProcessors()));
+        }
+    }
+
+    public static int getAccordCommandStoreShardCount()
+    {
+        return conf.accord.command_store_shard_count.or(DatabaseDescriptor::getAvailableProcessors);
+    }
+
+    public static int getAccordMaxQueuedLoadCount()
+    {
+        return conf.accord.max_queued_loads.or(getAccordConcurrentOps());
+    }
+
+    public static int getAccordMaxQueuedRangeLoadCount()
+    {
+        return conf.accord.max_queued_range_loads.or(Math.max(4, getAccordConcurrentOps() / 4));
+    }
+
+    public static boolean getAccordCacheShrinkingOn()
+    {
+        return conf.accord.shrink_cache_entries_before_eviction;
+    }
+
+    public static long getAccordRecoverDelay(TimeUnit units)
+    {
+        return conf.accord.recover_delay.to(units);
+    }
+
+    public static void setAccordRecoverDelay(long time, TimeUnit units)
+    {
+        conf.accord.recover_delay = new IntMillisecondsBound(time, units);
+    }
+
+    public static long getAccordRangeSyncPointRecoverDelay(TimeUnit units)
+    {
+        return conf.accord.range_syncpoint_recover_delay.to(units);
+    }
+
+    public static void setAccordRangeSyncPointRecoverDelay(long time, TimeUnit units)
+    {
+        conf.accord.range_syncpoint_recover_delay = new IntMillisecondsBound(time, units);
     }
 
     public static long getAccordFastPathUpdateDelayMillis()
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
index 3c7b113136..8a33147661 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
@@ -35,6 +35,7 @@ import com.google.common.collect.Ordering;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import accord.api.Agent;
 import accord.local.Cleanup;
 import accord.local.CommandStores;
 import accord.local.CommandStores.RangesForEpoch;
@@ -103,6 +104,7 @@ import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.SavedCommand;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.paxos.PaxosRepairHistory;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
@@ -805,6 +807,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
     class AccordCommandsPurger extends AbstractPurger
     {
+        final Agent agent;
         final Int2ObjectHashMap<RedundantBefore> redundantBefores;
         final Int2ObjectHashMap<DurableBefore> durableBefores;
         final Int2ObjectHashMap<RangesForEpoch> ranges;
@@ -814,7 +817,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
         AccordCommandsPurger(Supplier<IAccordService> accordService)
         {
-            IAccordService.CompactionInfo compactionInfo = accordService.get().getCompactionInfo();
+            IAccordService service = accordService.get();
+            IAccordService.CompactionInfo compactionInfo = service.getCompactionInfo();
+            this.agent = service.agent();
             this.redundantBefores = compactionInfo.redundantBefores;
             this.ranges = compactionInfo.ranges;
             this.durableBefores = compactionInfo.durableBefores;
@@ -852,7 +857,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
             if (saveStatus.is(Status.Invalidated))
                 return saveStatusOnly(saveStatus, row, nowInSec);
 
-            Cleanup cleanup = shouldCleanupPartial(txnId, saveStatus, durability, participants,
+            Cleanup cleanup = shouldCleanupPartial(agent, txnId, saveStatus, durability, participants,
                                                    redundantBefore, durableBefore);
             switch (cleanup)
             {
@@ -1019,6 +1024,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         final ColumnMetadata recordColumn;
         final ColumnMetadata versionColumn;
         final AccordService service;
+        final AccordAgent agent;
 
         JournalKey key = null;
         Object builder = null;
@@ -1036,6 +1042,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
             userVersion = service.journalConfiguration().userVersion();
             IAccordService.CompactionInfo compactionInfo = service.getCompactionInfo();
 
+            this.agent = service.agent();
             this.redundantBefores = compactionInfo.redundantBefores;
             this.ranges = compactionInfo.ranges;
             this.durableBefores = compactionInfo.durableBefores;
@@ -1102,7 +1109,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
                 RedundantBefore redundantBefore = redundantBefores.get(key.commandStoreId);
                 DurableBefore durableBefore = durableBefores.get(key.commandStoreId);
-                Cleanup cleanup = commandBuilder.shouldCleanup(redundantBefore, durableBefore);
+                Cleanup cleanup = commandBuilder.shouldCleanup(agent, redundantBefore, durableBefore);
                 if (cleanup == ERASE)
                     return PartitionUpdate.fullPartitionDelete(metadata(), partition.partitionKey(), maxSeenTimestamp, nowInSec).unfilteredIterator();
 
diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
index 42190e0c2e..3317b41978 100644
--- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java
+++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -30,6 +31,7 @@ import java.util.Objects;
 
 import javax.annotation.Nullable;
 
+import io.netty.util.concurrent.FastThreadLocal;
 import org.apache.cassandra.cql3.AssignmentTestable;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.ColumnSpecification;
@@ -739,6 +741,39 @@ public abstract class AbstractType<T> implements Comparator<ByteBuffer>, Assignm
             throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement asComparableBytes");
     }
 
+    protected static final FastThreadLocal<byte[]> tmpFlattenBuffer = new FastThreadLocal<>();
+    public static byte[] flattenByteSource(ByteSource source)
+    {
+        byte[] tmpBytes = tmpFlattenBuffer.get();
+        byte[] bytes = tmpBytes;
+        if (bytes == null) bytes = new byte[16];
+        int c = 0;
+        while (true)
+        {
+            int b = source.next();
+            if (b == ByteSource.END_OF_STREAM)
+                break;
+
+            if (c == bytes.length)
+                bytes = Arrays.copyOf(bytes, c * 2);
+
+            bytes[c++] = (byte)b;
+        }
+
+        byte[] result = Arrays.copyOf(bytes, c);
+        if (bytes != tmpBytes) tmpFlattenBuffer.set(bytes);
+        return result;
+    }
+
+    public <V> byte[] asFlatComparableBytes(ValueAccessor<V> accessor, V value, ByteComparable.Version version)
+    {
+        ByteSource source = asComparableBytes(accessor, value, version);
+        if (source == null)
+            return null;
+
+        return flattenByteSource(source);
+    }
+
     public final ByteSource asComparableBytes(ByteBuffer byteBuffer, ByteComparable.Version version)
     {
         return asComparableBytes(ByteBufferAccessor.instance, byteBuffer, version);
diff --git a/src/java/org/apache/cassandra/db/marshal/CompositeType.java b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
index df7ee99070..e175045049 100644
--- a/src/java/org/apache/cassandra/db/marshal/CompositeType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CompositeType.java
@@ -29,6 +29,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import accord.utils.Invariants;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.serializers.BytesSerializer;
@@ -39,8 +40,15 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
 import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
 
+import static accord.utils.Invariants.Paranoia.CONSTANT;
+import static accord.utils.Invariants.Paranoia.LINEAR;
+import static accord.utils.Invariants.ParanoiaCostFactor.LOW;
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.transform;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.END_OF_STREAM;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT_NULL;
+import static org.apache.cassandra.utils.bytecomparable.ByteSource.TERMINATOR;
 
 /*
  * The encoding of a CompositeType column name should be:
@@ -250,7 +258,64 @@ public class CompositeType extends AbstractCompositeType
         if (i * 2 + 1 < srcs.length)
             srcs = Arrays.copyOfRange(srcs, 0, i * 2 + 1);
 
-        return ByteSource.withTerminatorMaybeLegacy(version, ByteSource.END_OF_STREAM, srcs);
+        return ByteSource.withTerminatorMaybeLegacy(version, END_OF_STREAM, srcs);
+    }
+
+    @Override
+    public <V> byte[] asFlatComparableBytes(ValueAccessor<V> accessor, V data, Version version)
+    {
+        if (data == null || accessor.isEmpty(data))
+            return null;
+
+        byte[] tmpBytes = tmpFlattenBuffer.get();
+        byte[] bytes = tmpBytes;
+        if (bytes == null) bytes = new byte[16];
+
+        int c = 0;
+        int length = accessor.size(data);
+
+        // statics go first
+        boolean isStatic = readIsStaticInternal(data, accessor);
+        int offset = startingOffsetInternal(isStatic);
+        bytes[c++] = (byte) (isStatic ? NEXT_COMPONENT_NULL : NEXT_COMPONENT);
+        bytes[c++] = (byte) (NEXT_COMPONENT);
+
+        int i = 0;
+        byte lastEoc = 0;
+        while (offset < length)
+        {
+            // Only the end-of-component byte of the last component of this composite can be non-zero, so the
+            // component before can't have a non-zero end-of-component byte.
+            assert lastEoc == 0 : lastEoc;
+
+            int componentLength = accessor.getUnsignedShort(data, offset);
+            offset += 2;
+            ByteSource tmp = types.get(i).asComparableBytes(accessor, accessor.slice(data, offset, componentLength), version);
+            while (true)
+            {
+                int b = tmp.next();
+                if (b == END_OF_STREAM) break;
+
+                if (c == bytes.length) bytes = Arrays.copyOf(bytes, c * 2);
+                bytes[c++] = (byte)b;
+            }
+            offset += componentLength;
+            lastEoc = accessor.getByte(data, offset);
+            offset += 1;
+            if (c == bytes.length) bytes = Arrays.copyOf(bytes, c * 2);
+            bytes[c++] = (byte) NEXT_COMPONENT;
+            bytes[c++] = (byte) (lastEoc & 0xFF ^ 0x80); // end-of-component also takes part in comparison as signed byte
+            bytes[c++] = (byte) (offset < length ? NEXT_COMPONENT : version == Version.LEGACY ? END_OF_STREAM : TERMINATOR);
+            ++i;
+        }
+
+        byte[] result = Arrays.copyOf(bytes, c);
+        if (bytes != tmpBytes) tmpFlattenBuffer.set(bytes);
+        byte[] test = super.asFlatComparableBytes(accessor, data, version);
+        if (Invariants.isParanoid() && Invariants.testParanoia(LINEAR, CONSTANT, LOW)) Invariants.checkState(Arrays.equals(test, result));
+        V roundtrip = fromComparableBytes(accessor, ByteSource.peekable(ByteSource.of(result, version)), version);
+        Invariants.checkState(accessor.compare(data, roundtrip, accessor) == 0);
+        return result;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
index 2f2c2a2551..dd2254721b 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
@@ -40,6 +40,8 @@ import org.github.jamm.Unmetered;
 
 public abstract class AbstractMemtable implements Memtable
 {
+    private static final AtomicLong nextId = new AtomicLong();
+
     private final AtomicReference<LifecycleTransaction> flushTransaction = new AtomicReference<>(null);
     protected final AtomicLong currentOperations = new AtomicLong(0);
     protected final ColumnsCollector columnsCollector;
@@ -48,6 +50,7 @@ public abstract class AbstractMemtable implements Memtable
     protected AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE);
     // The smallest local deletion time for all partitions in this memtable
     protected AtomicLong minLocalDeletionTime = new AtomicLong(Long.MAX_VALUE);
+    private final long id = nextId.incrementAndGet();
     // Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not
     // correct for their usage.
 
@@ -80,6 +83,12 @@ public abstract class AbstractMemtable implements Memtable
         return currentOperations.get();
     }
 
+    @Override
+    public long getMemtableId()
+    {
+        return id;
+    }
+
     @Override
     public long getMinTimestamp()
     {
diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java
index b34a617ef4..dc0b7b3a1d 100644
--- a/src/java/org/apache/cassandra/db/memtable/Memtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java
@@ -179,6 +179,11 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
 
     // Main write and read operations
 
+    default long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    {
+        return put(update, indexer, opGroup, false);
+    }
+
     /**
      * Put new data in the memtable. This operation may block until enough memory is available in the memory pool.
      *
@@ -186,12 +191,14 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
      * @param indexer receives information about the update's effect
      * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a
      *                flush to free space.
+     * @param assumeMissing if true, the implementation MAY clone the key and attempt putIfAbsent without first
+     *                      looking for the keys' presence
      *
      * @return the smallest timestamp delta between corresponding rows from existing and update. A
      * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition
      * and those in {@code update}. See CASSANDRA-7979.
      */
-    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup);
+    long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing);
 
     // Read operations are provided by the UnfilteredSource interface.
 
@@ -363,6 +370,8 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
      */
     boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition);
 
+    long getMemtableId();
+
     /** Approximate commit log lower bound, <= getCommitLogLowerBound, used as a time stamp for ordering */
     CommitLogPosition getApproximateCommitLogLowerBound();
 
diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
index eb4a44ebd2..9b9a531a2b 100644
--- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java
@@ -138,11 +138,11 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
      *
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
         DecoratedKey key = update.partitionKey();
         MemtableShard shard = shards[boundaries.getShardForKey(key)];
-        return shard.put(key, update, indexer, opGroup);
+        return shard.put(key, update, indexer, opGroup, assumeMissing);
     }
 
     /**
@@ -366,10 +366,10 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
             this.metadata = metadata;
         }
 
-        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
         {
             Cloner cloner = allocator.cloner(opGroup);
-            AtomicBTreePartition previous = partitions.get(key);
+            AtomicBTreePartition previous = assumeMissing ? null : partitions.get(key);
 
             long initialSize = 0;
             if (previous == null)
@@ -504,13 +504,13 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable
          *
          * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
          */
-        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+        public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
         {
             DecoratedKey key = update.partitionKey();
             MemtableShard shard = shards[boundaries.getShardForKey(key)];
             synchronized (shard)
             {
-                return shard.put(key, update, indexer, opGroup);
+                return shard.put(key, update, indexer, opGroup, assumeMissing);
             }
         }
 
diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
index 3f6fbcbd52..985dd310fd 100644
--- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java
@@ -114,15 +114,14 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
     @Override
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
-        Cloner cloner = allocator.cloner(opGroup);
-        AtomicBTreePartition previous = partitions.get(update.partitionKey());
-
         long initialSize = 0;
+        Cloner cloner = allocator.cloner(opGroup);
+        AtomicBTreePartition previous = assumeMissing ? null : partitions.get(update.partitionKey());
         if (previous == null)
         {
-            final DecoratedKey cloneKey = cloner.clone(update.partitionKey());
+            DecoratedKey cloneKey = cloner.clone(update.partitionKey());
             AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator);
             // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent
             previous = partitions.putIfAbsent(cloneKey, empty);
diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
index a8fc54b891..2a2813d0ad 100644
--- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java
@@ -180,7 +180,7 @@ public class TrieMemtable extends AbstractShardedMemtable
      * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
      */
     @Override
-    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
+    public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing)
     {
         try
         {
diff --git a/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java b/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java
index 0918d73a5d..bd4fcc547f 100644
--- a/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java
+++ b/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java
@@ -36,12 +36,9 @@ import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import accord.local.CommandStores;
 import accord.primitives.Status;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.FieldIdentifier;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
@@ -57,10 +54,11 @@ import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.AccordCommandStore;
+import org.apache.cassandra.service.accord.AccordCache;
+import org.apache.cassandra.service.accord.AccordCommandStores;
+import org.apache.cassandra.service.accord.AccordExecutor;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordStateCache;
 import org.apache.cassandra.service.accord.CommandStoreTxnBlockedGraph;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
@@ -81,20 +79,20 @@ public class AccordVirtualTables
             return Collections.emptyList();
 
         return List.of(
-            new CommandStoreCache(keyspace),
+            new ExecutorCache(keyspace),
             new MigrationState(keyspace),
             new CoordinationStatus(keyspace),
             new TxnBlockedByTable(keyspace)
         );
     }
 
-    public static final class CommandStoreCache extends AbstractVirtualTable
+    public static final class ExecutorCache extends AbstractVirtualTable
     {
-        private CommandStoreCache(String keyspace)
+        private ExecutorCache(String keyspace)
         {
             super(parse(keyspace,
-                        "Accord Command Store Cache Metrics",
-                        "CREATE TABLE accord_command_store_cache(\n" +
+                        "Accord Executor Cache Metrics",
+                        "CREATE TABLE accord_executor_cache(\n" +
                         "  id int,\n" +
                         "  scope text,\n" +
                         "  queries bigint,\n" +
@@ -107,32 +105,22 @@ public class AccordVirtualTables
         @Override
         public DataSet data()
         {
-            CommandStores stores = ((AccordService) AccordService.instance()).node().commandStores();
-
-            AsyncChain<List<Map<String, AccordStateCache.ImmutableStats>>> statsByStoreChain = stores.map(store -> {
-                Map<String, AccordStateCache.ImmutableStats> snapshots = new HashMap<>(3);
-                AccordCommandStore accordStore = (AccordCommandStore) store.commandStore();
-                snapshots.put(AccordKeyspace.COMMANDS, accordStore.commandCache().statsSnapshot());
-                snapshots.put(AccordKeyspace.COMMANDS_FOR_KEY, accordStore.commandsForKeyCache().statsSnapshot());
-                snapshots.put(AccordKeyspace.TIMESTAMPS_FOR_KEY, accordStore.timestampsForKeyCache().statsSnapshot());
-                return snapshots;
-            });
-
-            List<Map<String, AccordStateCache.ImmutableStats>> statsByStore = AsyncChains.getBlockingAndRethrow(statsByStoreChain);
+            AccordCommandStores stores = (AccordCommandStores) ((AccordService) AccordService.instance()).node().commandStores();
             SimpleDataSet result = new SimpleDataSet(metadata());
-
-            for (int storeID : stores.ids())
+            for (AccordExecutor executor : stores.executors())
             {
-                Map<String, AccordStateCache.ImmutableStats> storeStats = statsByStore.get(storeID);
-                addRow(storeStats.get(AccordKeyspace.COMMANDS), result, storeID, AccordKeyspace.COMMANDS);
-                addRow(storeStats.get(AccordKeyspace.COMMANDS_FOR_KEY), result, storeID, AccordKeyspace.COMMANDS_FOR_KEY);
-                addRow(storeStats.get(AccordKeyspace.TIMESTAMPS_FOR_KEY), result, storeID, AccordKeyspace.TIMESTAMPS_FOR_KEY);
+                Map<String, AccordCache.ImmutableStats> snapshots = new HashMap<>(3);
+                try (AccordExecutor.ExclusiveGlobalCaches cache = executor.lockCaches())
+                {
+                    addRow(cache.commands.statsSnapshot(), result, executor.executorId(), AccordKeyspace.COMMANDS);
+                    addRow(cache.commandsForKey.statsSnapshot(), result, executor.executorId(), AccordKeyspace.COMMANDS_FOR_KEY);
+                    addRow(cache.timestampsForKey.statsSnapshot(), result, executor.executorId(), AccordKeyspace.TIMESTAMPS_FOR_KEY);
+                }
             }
-
             return result;
         }
 
-        private static void addRow(AccordStateCache.ImmutableStats stats, SimpleDataSet result, int storeID, String scope)
+        private static void addRow(AccordCache.ImmutableStats stats, SimpleDataSet result, int storeID, String scope)
         {
             result.row(storeID, scope);
             result.column("queries", stats.queries);
diff --git a/src/java/org/apache/cassandra/dht/LocalPartitioner.java b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
index 2f060c6b67..4c45887dc4 100644
--- a/src/java/org/apache/cassandra/dht/LocalPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
@@ -186,7 +186,6 @@ public class LocalPartitioner implements IPartitioner
         {
             // todo (tcm); seems partitioner got mutated on alter type (for example) before tcm, now we create a new one - not sure its enough just making sure that its the same type of partitioner
             assert o.getPartitioner().getClass().equals(getPartitioner().getClass()) : String.format("partitioners do not match; %s != %s", getPartitioner(), o.getPartitioner());
-//            assert getPartitioner() == o.getPartitioner() : String.format("partitioners do not match; %s != %s", getPartitioner(), o.getPartitioner());
             return comparator.compare(token, ((LocalToken) o).token);
         }
 
diff --git a/src/java/org/apache/cassandra/hints/HintsBuffer.java b/src/java/org/apache/cassandra/hints/HintsBuffer.java
index 646dd72feb..41c9cdf234 100644
--- a/src/java/org/apache/cassandra/hints/HintsBuffer.java
+++ b/src/java/org/apache/cassandra/hints/HintsBuffer.java
@@ -179,7 +179,19 @@ final class HintsBuffer
         return new Allocation(offset, totalSize, opGroup);
     }
 
-    // allocate bytes in the slab, or return negative if not enough space
+    /**
+     * Allocate bytes in the segment, or return -1 if not enough space. Method ensures that marker bytes
+     * for each allocation (i.e. offset of its end) is written as a 32 bit integer at its beginning, and
+     * that these marker bytes are always written sequentially. In other words, if allocation A has a lower
+     * starting offset than allocation B, A's marker will always be written before the offset for B is returned.
+     *
+     * `allocateOffset` consists of two integers:
+     *    64                 32                0
+     *    | (i32) inProgress | (i32) writtenTo |
+     *
+     *  If inProgress bytes are not zeroes, they contain an unwritten offset. Before allocating any bytes,
+     *  inProgresss bytes need to be written at the writtenTo location in the target buffer.
+     */
     private int allocateBytes(int totalSize)
     {
         long prev = position.getAndAdd(totalSize);
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index bd3de9521e..e3ae1907d7 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -239,6 +239,14 @@ public final class HintsService implements HintsServiceMBean
         writeExecutor.fsyncWritersBlockingly(stores);
     }
 
+    @VisibleForTesting
+    public void flushAndFsyncBlockingly()
+    {
+        List<HintsStore> stores = catalog.stores().collect(Collectors.toList());
+        writeExecutor.flushBufferPool(bufferPool, stores);
+        writeExecutor.fsyncWritersBlockingly(stores);
+    }
+
     public synchronized void startDispatch()
     {
         if (isShutDown)
diff --git a/src/java/org/apache/cassandra/index/accord/RoutesSearcher.java b/src/java/org/apache/cassandra/index/accord/RoutesSearcher.java
index cade5d28ef..363a3a4d99 100644
--- a/src/java/org/apache/cassandra/index/accord/RoutesSearcher.java
+++ b/src/java/org/apache/cassandra/index/accord/RoutesSearcher.java
@@ -19,12 +19,10 @@
 package org.apache.cassandra.index.accord;
 
 import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Set;
+import java.util.function.Consumer;
 
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
-import org.agrona.collections.ObjectHashSet;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
@@ -48,7 +46,7 @@ import org.apache.cassandra.utils.FBUtilities;
 public class RoutesSearcher
 {
     private final ColumnFamilyStore cfs = Keyspace.open("system_accord").getColumnFamilyStore("commands");
-    private final Index index = cfs.indexManager.getIndexByName("route");;
+    private final Index index = cfs.indexManager.getIndexByName("route");
     private final ColumnMetadata participants = AccordKeyspace.CommandsColumns.participants;
     private final ColumnMetadata store_id = AccordKeyspace.CommandsColumns.store_id;
     private final ColumnMetadata txn_id = AccordKeyspace.CommandsColumns.txn_id;
@@ -101,14 +99,13 @@ public class RoutesSearcher
         }
     }
 
-    public Set<TxnId> intersects(int store, TokenRange range, TxnId minTxnId, Timestamp maxTxnId)
+    public void intersects(int store, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, Consumer<TxnId> forEach)
     {
-        return intersects(store, range.start(), range.end(), minTxnId, maxTxnId);
+        intersects(store, range.start(), range.end(), minTxnId, maxTxnId, forEach);
     }
 
-    public Set<TxnId> intersects(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId)
+    void intersects(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId, Consumer<TxnId> forEach)
     {
-        ObjectHashSet<TxnId> set = new ObjectHashSet<TxnId>();
         try (CloseableIterator<Entry> it = searchKeysAccord(store, start, end))
         {
             while (it.hasNext())
@@ -116,10 +113,9 @@ public class RoutesSearcher
                 Entry next = it.next();
                 if (next.store_id != store) continue; // the index should filter out, but just in case...
                 if (next.txnId.compareTo(minTxnId) >= 0 && next.txnId.compareTo(maxTxnId) < 0)
-                    set.add(next.txnId);
+                    forEach.accept(next.txnId);
             }
         }
-        return set.isEmpty() ? Collections.emptySet() : set;
     }
 
     private static final class Entry
diff --git a/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java b/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
index 3cb5db0f00..837e10c1b6 100644
--- a/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
+++ b/src/java/org/apache/cassandra/io/util/DataOutputBuffer.java
@@ -60,7 +60,7 @@ public class DataOutputBuffer extends BufferedDataOutputStreamPlus
      * Scratch buffers used mostly for serializing in memory. It's important to call #close() when finished
      * to keep the memory overhead from being too large in the system.
      */
-    public static final FastThreadLocal<DataOutputBuffer> scratchBuffer = new FastThreadLocal<DataOutputBuffer>()
+    public static final FastThreadLocal<DataOutputBuffer> scratchBuffer = new FastThreadLocal<>()
     {
         @Override
         protected DataOutputBuffer initialValue()
diff --git a/src/java/org/apache/cassandra/journal/ActiveSegment.java b/src/java/org/apache/cassandra/journal/ActiveSegment.java
index 69f2f3323c..f9ff3fd937 100644
--- a/src/java/org/apache/cassandra/journal/ActiveSegment.java
+++ b/src/java/org/apache/cassandra/journal/ActiveSegment.java
@@ -24,6 +24,7 @@ import java.nio.channels.FileChannel;
 import java.nio.file.StandardOpenOption;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.locks.LockSupport;
 
 import com.codahale.metrics.Timer;
@@ -44,16 +45,18 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     private final OpOrder appendOrder = new OpOrder();
 
     // position in the buffer we are allocating from
-    private volatile int allocateOffset = 0;
-    private static final AtomicIntegerFieldUpdater<ActiveSegment> allocateOffsetUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "allocateOffset");
+    private volatile long allocateOffset = 0;
+    private static final AtomicLongFieldUpdater<ActiveSegment> allocateOffsetUpdater = AtomicLongFieldUpdater.newUpdater(ActiveSegment.class, "allocateOffset");
 
     /*
      * Everything before this offset has been written and flushed.
      */
-    private volatile int lastFlushedOffset = 0;
-    private volatile int lastFsyncOffset = 0;
+    private volatile int writtenTo = 0;
+    private volatile int fsyncedTo = 0;
     @SuppressWarnings("rawtypes")
-    private static final AtomicIntegerFieldUpdater<ActiveSegment> lastFsyncOffsetUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "lastFsyncOffset");
+    private static final AtomicIntegerFieldUpdater<ActiveSegment> writtenToUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "writtenTo");
+    @SuppressWarnings("rawtypes")
+    private static final AtomicIntegerFieldUpdater<ActiveSegment> fsyncedToUpdater = AtomicIntegerFieldUpdater.newUpdater(ActiveSegment.class, "fsyncedTo");
 
     /*
      * End position of the buffer; initially set to its capacity and
@@ -70,16 +73,16 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     final InMemoryIndex<K> index;
 
     private ActiveSegment(
-        Descriptor descriptor, Params params, SyncedOffsets syncedOffsets, InMemoryIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
+        Descriptor descriptor, Params params, InMemoryIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
     {
-        super(descriptor, syncedOffsets, metadata, keySupport);
+        super(descriptor, metadata, keySupport);
         this.index = index;
         try
         {
             channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE);
             buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, params.segmentSize());
             endOfBuffer = buffer.capacity();
-            selfRef = new Ref<>(this, new Tidier(descriptor, channel, buffer, syncedOffsets));
+            selfRef = new Ref<>(this, new Tidier(descriptor, channel, buffer));
         }
         catch (IOException e)
         {
@@ -87,13 +90,11 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    @SuppressWarnings("resource")
     static <K, V> ActiveSegment<K, V> create(Descriptor descriptor, Params params, KeySupport<K> keySupport)
     {
-        SyncedOffsets syncedOffsets = SyncedOffsets.active(descriptor);
         InMemoryIndex<K> index = InMemoryIndex.create(keySupport);
         Metadata metadata = Metadata.create();
-        return new ActiveSegment<>(descriptor, params, syncedOffsets, index, metadata, keySupport);
+        return new ActiveSegment<>(descriptor, params, index, metadata, keySupport);
     }
 
     @Override
@@ -147,40 +148,40 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     /**
      * Stop writing to this file, flush and close it. Does nothing if the file is already closed.
      */
-    @Override
-    public synchronized void close()
+    public synchronized void close(Journal<K, V> journal)
     {
-        close(true);
+        close(journal, true);
     }
 
     /**
      * @return true if the closed segment was definitely empty, false otherwise
      */
-    private synchronized boolean close(boolean persistComponents)
+    private synchronized boolean close(Journal<K, V> journal, boolean persistComponents)
     {
         boolean isEmpty = discardUnusedTail();
         if (!isEmpty)
         {
-            flush(true);
+            updateWrittenTo();
+            fsync();
             if (persistComponents) persistComponents();
         }
-        release();
+        release(journal);
         return isEmpty;
     }
 
     /**
      * Close and discard a pre-allocated, available segment, that's never been exposed
      */
-    void closeAndDiscard()
+    void closeAndDiscard(Journal<K, V> journal)
     {
-        boolean isEmpty = close(false);
+        boolean isEmpty = close(journal, false);
         if (!isEmpty) throw new IllegalStateException();
         discard();
     }
 
-    void closeAndIfEmptyDiscard()
+    void closeAndIfEmptyDiscard(Journal<K, V> journal)
     {
-        boolean isEmpty = close(true);
+        boolean isEmpty = close(journal, true);
         if (isEmpty) discard();
     }
 
@@ -188,7 +189,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     {
         index.persist(descriptor);
         metadata.persist(descriptor);
-        syncedOffsets.fsync();
         SyncUtil.trySyncDir(descriptor.directory);
     }
 
@@ -199,13 +199,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         descriptor.fileFor(Component.DATA).deleteIfExists();
         descriptor.fileFor(Component.INDEX).deleteIfExists();
         descriptor.fileFor(Component.METADATA).deleteIfExists();
-        descriptor.fileFor(Component.SYNCED_OFFSETS).deleteIfExists();
-    }
-
-    @Override
-    void release()
-    {
-        selfRef.release();
     }
 
     @Override
@@ -220,23 +213,27 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         return selfRef.ref();
     }
 
-    private static final class Tidier implements Tidy
+    @Override
+    public Ref<Segment<K, V>> selfRef()
+    {
+        return selfRef;
+    }
+
+    private static final class Tidier extends Segment.Tidier implements Tidy
     {
         private final Descriptor descriptor;
         private final FileChannel channel;
         private final ByteBuffer buffer;
-        private final SyncedOffsets syncedOffsets;
 
-        Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer, SyncedOffsets syncedOffsets)
+        Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer)
         {
             this.descriptor = descriptor;
             this.channel = channel;
             this.buffer = buffer;
-            this.syncedOffsets = syncedOffsets;
         }
 
         @Override
-        public void tidy()
+        void onUnreferenced()
         {
             FileUtils.clean(buffer);
             try
@@ -247,7 +244,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             {
                 throw new JournalWriteError(descriptor, Component.DATA, e);
             }
-            syncedOffsets.close();
         }
 
         @Override
@@ -257,68 +253,49 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    /*
-     * Flush logic; closing and component flushing
-     */
-
-    boolean shouldFlush()
-    {
-        int allocateOffset = this.allocateOffset;
-        return lastFlushedOffset < allocateOffset;
-    }
-
     public boolean isFlushed(long position)
     {
-        return lastFlushedOffset >= position;
+        return writtenTo >= position;
     }
 
-    public long lastFlushedOffset()
+    public int writtenToAtLeast()
     {
-        return lastFlushedOffset;
+        return writtenTo;
     }
 
-    /**
-     * Possibly force a disk flush for this segment file.
-     * TODO FIXME: calls from outside Flusher + callbacks
-     * @return last synced offset
-     */
-    synchronized int flush(boolean fsync)
+    public int fsyncedTo()
     {
-        int allocateOffset = this.allocateOffset;
-        if (lastFlushedOffset >= allocateOffset)
-            return lastFlushedOffset;
+        return fsyncedTo;
+    }
+
+    public int updateWrittenTo()
+    {
+        int allocatedTo = (int)allocateOffset;
+        if (writtenTo >= allocatedTo)
+            return writtenTo;
 
         waitForModifications();
-        if (fsync)
-        {
-            fsyncInternal();
-            lastFsyncOffsetUpdater.accumulateAndGet(this, allocateOffset, Math::max);
-        }
-        lastFlushedOffset = allocateOffset;
-        int syncedOffset = Math.min(allocateOffset, endOfBuffer);
-        syncedOffsets.mark(syncedOffset, fsync);
-        flushComplete.signalAll();
-        return syncedOffset;
+        return writtenToUpdater.accumulateAndGet(this, allocatedTo, Math::max);
     }
 
     // provides no ordering guarantees
     void fsync()
     {
-        int lastFlushed = lastFlushedOffset;
-        if (lastFsyncOffset >= lastFlushed)
+        int writtenTo = this.writtenTo;
+        if (fsyncedTo >= writtenTo)
             return;
 
         fsyncInternal();
-        syncedOffsets.fsync();
-        lastFsyncOffsetUpdater.accumulateAndGet(this, lastFlushed, Math::max);
+        fsyncedToUpdater.accumulateAndGet(this, writtenTo, Math::max);
+        flushComplete.signalAll();
     }
 
     private void waitForFlush(int position)
     {
-        while (lastFlushedOffset < position)
+        while (fsyncedTo < position)
         {
             WaitQueue.Signal signal = flushComplete.register();
-            if (lastFlushedOffset < position)
+            if (fsyncedTo < position)
                 signal.awaitThrowUncheckedOnInterrupt();
             else
                 signal.cancel();
@@ -346,12 +323,6 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    boolean isFullyFlushed()
-    {
-        int allocateOffset = this.allocateOffset;
-        return lastFsyncOffset >= allocateOffset;
-    }
-
     /**
      * Ensures no more of this segment is writeable, by allocating any unused section at the end
      * and marking it discarded void discartUnusedTail()
@@ -364,10 +335,10 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         {
             while (true)
             {
-                int prev = allocateOffset;
+                long prev = completeInProgress();
                 int next = endOfBuffer + 1;
 
-                if (prev >= next)
+                if ((int)prev >= next)
                 {
                     // already stopped allocating, might also be closed
                     assert buffer == null || prev == buffer.capacity() + 1;
@@ -377,10 +348,11 @@ final class ActiveSegment<K, V> extends Segment<K, V>
                 if (allocateOffsetUpdater.compareAndSet(this, prev, next))
                 {
                     // stopped allocating now; can only succeed once, no further allocation or discardUnusedTail can succeed
-                    endOfBuffer = prev;
+                    endOfBuffer = (int)prev;
                     assert buffer != null && next == buffer.capacity() + 1;
                     return prev == 0;
                 }
+                LockSupport.parkNanos(1);
             }
         }
     }
@@ -414,7 +386,8 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     private int totalEntrySize(Set<Integer> hosts, int recordSize)
     {
         return EntrySerializer.fixedEntrySize(keySupport, descriptor.userVersion)
-             + EntrySerializer.variableEntrySize(hosts.size(), recordSize);
+             + EntrySerializer.variableEntrySize(hosts.size())
+               + recordSize;
     }
 
     // allocate bytes in the segment, or return -1 if not enough space
@@ -422,16 +395,30 @@ final class ActiveSegment<K, V> extends Segment<K, V>
     {
         while (true)
         {
-            int prev = allocateOffset;
-            int next = prev + size;
+            long prev = maybeCompleteInProgress();
+            if (prev < 0)
+            {
+                LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+                continue;
+            }
+
+            long next = prev + size;
             if (next >= endOfBuffer)
                 return -1;
-            if (allocateOffsetUpdater.compareAndSet(this, prev, next))
+
+            // TODO (expected): if we write a "safe shutdown" marker we don't need this,
+            //  but this provides safe restart in the event the process terminates abruptly but the host remains stable
+            long inProgress = prev | (next << 32);
+            if (!allocateOffsetUpdater.compareAndSet(this, prev, inProgress))
             {
-                assert buffer != null;
-                return prev;
+                LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+                continue;
             }
-            LockSupport.parkNanos(1); // ConstantBackoffCAS Algorithm from https://arxiv.org/pdf/1305.5800.pdf
+
+            assert buffer != null;
+            buffer.putInt((int)prev, (int)next);
+            allocateOffsetUpdater.compareAndSet(this, inProgress, next);
+            return (int) prev;
         }
     }
 
@@ -450,14 +437,13 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             this.length = length;
         }
 
-        RecordPointer write(K id, ByteBuffer record, Set<Integer> hosts)
+        void write(K id, ByteBuffer record, Set<Integer> hosts)
         {
-            try (BufferedDataOutputStreamPlus out = new DataOutputBufferFixed(buffer))
+            try
             {
-                EntrySerializer.write(id, record, hosts, keySupport, out, descriptor.userVersion);
-                index.update(id, start, length);
+                EntrySerializer.write(id, record, hosts, keySupport, buffer, descriptor.userVersion);
                 metadata.update(hosts);
-                return new RecordPointer(descriptor.timestamp, start);
+                index.update(id, start, length);
             }
             catch (IOException e)
             {
@@ -472,9 +458,9 @@ final class ActiveSegment<K, V> extends Segment<K, V>
         // Variant of write that does not allocate/return a record pointer
         void writeInternal(K id, ByteBuffer record, Set<Integer> hosts)
         {
-            try (BufferedDataOutputStreamPlus out = new DataOutputBufferFixed(buffer))
+            try
             {
-                EntrySerializer.write(id, record, hosts, keySupport, out, descriptor.userVersion);
+                EntrySerializer.write(id, record, hosts, keySupport, buffer, descriptor.userVersion);
                 index.update(id, start, length);
                 metadata.update(hosts);
             }
@@ -488,12 +474,48 @@ final class ActiveSegment<K, V> extends Segment<K, V>
             }
         }
 
-        void awaitFlush(Timer waitingOnFlush)
+        void awaitDurable(Timer waitingOnFlush)
         {
             try (Timer.Context ignored = waitingOnFlush.time())
             {
                 waitForFlush(start);
             }
         }
+
+        boolean isFsynced()
+        {
+            return fsyncedTo >= start + length;
+        }
+
+        Descriptor descriptor()
+        {
+            return descriptor;
+        }
+
+        int start()
+        {
+            return start;
+        }
+    }
+
+    private int maybeCompleteInProgress()
+    {
+        long cur = allocateOffset;
+        int inProgress = (int) (cur >>> 32);
+        if (inProgress == 0) return (int) cur;
+        // finish up the in-progress allocation
+        buffer.putInt((int)cur, inProgress);
+        if (!allocateOffsetUpdater.compareAndSet(this, cur, inProgress))
+            return -1;
+
+        return inProgress;
+    }
+
+    private int completeInProgress()
+    {
+        int result = maybeCompleteInProgress();
+        while (result < 0)
+            result = maybeCompleteInProgress();
+        return result;
     }
 }
diff --git a/src/java/org/apache/cassandra/journal/Compactor.java b/src/java/org/apache/cassandra/journal/Compactor.java
index 4ecfb74091..0062b7471d 100644
--- a/src/java/org/apache/cassandra/journal/Compactor.java
+++ b/src/java/org/apache/cassandra/journal/Compactor.java
@@ -46,7 +46,7 @@ public final class Compactor<K, V> implements Runnable, Shutdownable
     synchronized void start()
     {
         if (journal.params.enableCompaction())
-            schedule(journal.params.compactionPeriodMillis(), TimeUnit.MILLISECONDS);
+            schedule(journal.params.compactionPeriod(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
     }
 
     private synchronized void schedule(long period, TimeUnit units)
@@ -85,7 +85,7 @@ public final class Compactor<K, V> implements Runnable, Shutdownable
 
             journal.replaceCompactedSegments(toCompact, newSegments);
             for (StaticSegment<K, V> segment : toCompact)
-                segment.discard();
+                segment.discard(journal);
         }
         catch (IOException e)
         {
diff --git a/src/java/org/apache/cassandra/journal/Component.java b/src/java/org/apache/cassandra/journal/Component.java
index c9de451adb..f7cf944f86 100644
--- a/src/java/org/apache/cassandra/journal/Component.java
+++ b/src/java/org/apache/cassandra/journal/Component.java
@@ -17,15 +17,19 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.List;
+
+import static accord.utils.SortedArrays.SortedArrayList.ofSorted;
+
 enum Component
 {
     DATA           ("data"),
     INDEX          ("indx"),
-    METADATA       ("meta"),
-    SYNCED_OFFSETS ("sync");
+    METADATA       ("meta");
     //OFFSET_MAP     (".offs"),
     //INVLALIDATIONS (".invl");
 
+    public static final List<Component> VALUES = ofSorted(values());
     final String extension;
 
     Component(String extension)
diff --git a/src/java/org/apache/cassandra/journal/EntrySerializer.java b/src/java/org/apache/cassandra/journal/EntrySerializer.java
index 2a707e7d73..a90cfa38f0 100644
--- a/src/java/org/apache/cassandra/journal/EntrySerializer.java
+++ b/src/java/org/apache/cassandra/journal/EntrySerializer.java
@@ -23,17 +23,13 @@ import java.nio.ByteBuffer;
 import java.util.Set;
 import java.util.zip.CRC32;
 
+import accord.utils.Invariants;
 import org.agrona.collections.IntHashSet;
 import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Crc;
 
 import static org.apache.cassandra.journal.Journal.validateCRC;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksum;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumShort;
 
 public final class EntrySerializer
 {
@@ -41,175 +37,172 @@ public final class EntrySerializer
                           ByteBuffer record,
                           Set<Integer> hosts,
                           KeySupport<K> keySupport,
-                          DataOutputPlus out,
+                          ByteBuffer out,
                           int userVersion)
     throws IOException
     {
-        CRC32 crc = Crc.crc32();
+        int start = out.position();
+        int totalSize = out.getInt() - start;
+        Invariants.checkState(totalSize == out.remaining() + TypeSizes.INT_SIZE);
+        Invariants.checkState(totalSize == record.remaining() + fixedEntrySize(keySupport, userVersion) + variableEntrySize(hosts.size()));
 
         keySupport.serialize(key, out, userVersion);
-        keySupport.updateChecksum(crc, key, userVersion);
+        out.putShort((short)hosts.size());
 
-        out.writeShort(hosts.size());
-        updateChecksumShort(crc, (short) hosts.size());
-
-        int recordSize = record.remaining();
-        out.writeInt(recordSize);
-        updateChecksumInt(crc, recordSize);
-
-        out.writeInt((int) crc.getValue());
+        int fixedCrcPosition = out.position();
+        out.position(fixedCrcPosition + TypeSizes.INT_SIZE);
 
         for (int host : hosts)
-        {
-            out.writeInt(host);
-            updateChecksumInt(crc, host);
-        }
+            out.putInt(host);
 
-        out.write(record);
-        Crc.updateCrc32(crc, record, record.position(), record.limit());
+        int recordSize = record.remaining();
+        int recordEnd = out.position() + recordSize;
+        Invariants.checkState(out.limit() == recordEnd + TypeSizes.INT_SIZE);
+        ByteBufferUtil.copyBytes(record, record.position(), out, out.position(), recordSize);
 
-        out.writeInt((int) crc.getValue());
+        // update and write crcs
+        CRC32 crc = Crc.crc32();
+        out.position(start);
+        out.limit(fixedCrcPosition);
+        crc.update(out);
+        out.limit(recordEnd);
+        out.putInt((int) crc.getValue());
+        crc.update(out);
+        out.limit(recordEnd + 4);
+        out.putInt((int) crc.getValue());
     }
 
+    // we reuse record as the value we return
     static <K> void read(EntryHolder<K> into,
                          KeySupport<K> keySupport,
                          ByteBuffer from,
                          int userVersion)
     throws IOException
     {
-        CRC32 crc = Crc.crc32();
         into.clear();
 
-        try (DataInputBuffer in = new DataInputBuffer(from, false))
+        int start = from.position();
         {
-            K key = keySupport.deserialize(in, userVersion);
-            keySupport.updateChecksum(crc, key, userVersion);
-            into.key = key;
+            int totalSize = from.getInt(start) - start;
+            Invariants.checkState(totalSize == from.remaining());
 
-            int hostCount = in.readShort();
-            updateChecksumShort(crc, (short) hostCount);
+            CRC32 crc = Crc.crc32();
+            int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
+            int fixedCrc = readAndUpdateFixedCrc(crc, from, fixedSize);
+            validateCRC(crc, fixedCrc);
 
-            int entrySize = in.readInt();
-            updateChecksumInt(crc, entrySize);
-
-            validateCRC(crc, in.readInt());
-
-            for (int i = 0; i < hostCount; i++)
-            {
-                int hostId = in.readInt();
-                updateChecksumInt(crc, hostId);
-                into.hosts.add(hostId);
-            }
-
-            // TODO: try to avoid allocating another buffer here
-            ByteBuffer entry = ByteBufferUtil.read(in, entrySize);
-            updateChecksum(crc, entry);
-            into.value = entry;
-            into.userVersion = userVersion;
-
-            validateCRC(crc, in.readInt());
+            int recordCrc = readAndUpdateRecordCrc(crc, from, start + totalSize);
+            validateCRC(crc, recordCrc);
         }
+
+        readValidated(into, from, start, keySupport, userVersion);
     }
 
-    static <K> boolean tryRead(EntryHolder<K> into,
-                               KeySupport<K> keySupport,
-                               ByteBuffer from,
-                               DataInputBuffer in,
-                               int syncedOffset,
-                               int userVersion)
+    // slices the provided buffer to assign to into.value
+    static <K> int tryRead(EntryHolder<K> into,
+                           KeySupport<K> keySupport,
+                           ByteBuffer from,
+                           int syncedOffset,
+                           int userVersion)
     throws IOException
     {
         CRC32 crc = Crc.crc32();
         into.clear();
 
-        int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
-        if (from.remaining() < fixedSize)
+        int start = from.position();
+        if (from.remaining() < TypeSizes.INT_SIZE)
+            return -1;
+
+        int totalSize = from.getInt(start) - start;
+        if (totalSize == 0)
+            return -1;
+
+        if (from.remaining() < totalSize)
             return handleReadException(new EOFException(), from.limit(), syncedOffset);
 
-        updateChecksum(crc, from, from.position(), fixedSize - TypeSizes.INT_SIZE);
-        int fixedCrc = from.getInt(from.position() + fixedSize - TypeSizes.INT_SIZE);
+        {
+            int fixedSize = EntrySerializer.fixedEntrySize(keySupport, userVersion);
+            int fixedCrc = readAndUpdateFixedCrc(crc, from, fixedSize);
+            try
+            {
+                validateCRC(crc, fixedCrc);
+            }
+            catch (IOException e)
+            {
+                return handleReadException(e, from.position() + fixedSize, syncedOffset);
+            }
 
-        try
-        {
-            validateCRC(crc, fixedCrc);
-        }
-        catch (IOException e)
-        {
-            return handleReadException(e, from.position() + fixedSize, syncedOffset);
+            int recordCrc = readAndUpdateRecordCrc(crc, from, start + totalSize);
+            try
+            {
+                validateCRC(crc, recordCrc);
+            }
+            catch (IOException e)
+            {
+                return handleReadException(e, from.position(), syncedOffset);
+            }
         }
 
-        int hostCount, recordSize;
-        try
-        {
-            into.key = keySupport.deserialize(in, userVersion);
-            hostCount = in.readShort();
-            recordSize = in.readInt();
-            in.skipBytesFully(TypeSizes.INT_SIZE);
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException(); // can't happen unless deserializer is buggy
-        }
-
-        int variableSize = EntrySerializer.variableEntrySize(hostCount, recordSize);
-        if (from.remaining() < variableSize)
-            return handleReadException(new EOFException(), from.limit(), syncedOffset);
-
-        updateChecksum(crc, from, from.position(), variableSize - TypeSizes.INT_SIZE);
-        int variableCrc = from.getInt(from.position() + variableSize - TypeSizes.INT_SIZE);
-
-        try
-        {
-            validateCRC(crc, variableCrc);
-        }
-        catch (IOException e)
-        {
-            return handleReadException(e, from.position() + variableSize, syncedOffset);
-        }
-
-        for (int i = 0; i < hostCount; i++)
-        {
-            into.hosts.add(in.readInt());
-        }
-
-        try
-        {
-            in.skipBytesFully(recordSize);
-        }
-        catch (IOException e)
-        {
-            throw new AssertionError(); // can't happen
-        }
-
-        into.value = from.duplicate()
-                         .position(from.position() - recordSize)
-                         .limit(from.position());
-        into.userVersion = userVersion;
-
-        in.skipBytesFully(TypeSizes.INT_SIZE);
-        return true;
+        readValidated(into, from, start, keySupport, userVersion);
+        return totalSize;
     }
 
-    private static boolean handleReadException(IOException e, int bufferPosition, int fsyncedLimit) throws IOException
+    private static <K> void readValidated(EntryHolder<K> into, ByteBuffer from, int start, KeySupport<K> keySupport, int userVersion)
+    {
+        from.position(start + TypeSizes.INT_SIZE);
+        into.key = keySupport.deserialize(from, userVersion);
+        int hostCount = from.getShort();
+
+        from.position(from.position() + 4);
+        for (int i = 0; i < hostCount; i++)
+        {
+            int hostId = from.getInt();
+            into.hosts.add(hostId);
+        }
+
+        into.value = from;
+        into.userVersion = userVersion;
+    }
+
+    private static int readAndUpdateFixedCrc(CRC32 crc, ByteBuffer from, int fixedSize)
+    {
+        int fixedEnd = from.position() + fixedSize - TypeSizes.INT_SIZE;
+        int fixedCrc = from.getInt(fixedEnd);
+        from.limit(fixedEnd);
+        crc.update(from);
+        return fixedCrc;
+    }
+
+    private static int readAndUpdateRecordCrc(CRC32 crc, ByteBuffer from, int limit)
+    {
+        int recordEnd = limit - TypeSizes.INT_SIZE;
+        from.limit(limit);
+        int recordCrc = from.getInt(recordEnd);
+        from.position(from.position() + 4);
+        from.limit(recordEnd);
+        crc.update(from);
+        return recordCrc;
+    }
+
+    private static int handleReadException(IOException e, int bufferPosition, int fsyncedLimit) throws IOException
     {
         if (bufferPosition <= fsyncedLimit)
             throw e;
         else
-            return false;
+            return -1;
     }
 
     static <K> int fixedEntrySize(KeySupport<K> keySupport, int userVersion)
     {
         return keySupport.serializedSize(userVersion) // key/id
              + TypeSizes.SHORT_SIZE                   // host count
-             + TypeSizes.INT_SIZE                     // record size
+             + TypeSizes.INT_SIZE                     // total size
              + TypeSizes.INT_SIZE;                    // CRC
     }
 
-    static int variableEntrySize(int hostCount, int recordSize)
+    static int variableEntrySize(int hostCount)
     {
         return TypeSizes.INT_SIZE * hostCount // hosts
-             + recordSize                     // record
              + TypeSizes.INT_SIZE;            // CRC
     }
 
diff --git a/src/java/org/apache/cassandra/journal/Flusher.java b/src/java/org/apache/cassandra/journal/Flusher.java
index 7982dcec7c..2aba68d8ea 100644
--- a/src/java/org/apache/cassandra/journal/Flusher.java
+++ b/src/java/org/apache/cassandra/journal/Flusher.java
@@ -45,7 +45,6 @@ import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe
 import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
 import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN;
 import static org.apache.cassandra.journal.Params.FlushMode.PERIODIC;
-import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized;
 import static org.apache.cassandra.utils.MonotonicClock.Global.preciseTime;
 import static org.apache.cassandra.utils.Simulate.With.GLOBAL_CLOCK;
@@ -69,35 +68,36 @@ final class Flusher<K, V>
     private final AtomicLong written = new AtomicLong(0);
 
     // the time of the last initiated flush
-    volatile long flushStartedAt = nanoTime();
+    volatile long flushStartedAt;
     // the time of the earliest flush that has completed an fsync; all Allocations written before this time are durable
     volatile long fsyncFinishedFor = flushStartedAt;
+    volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0);
 
     // a signal that writers can wait on to be notified of a completed flush in PERIODIC FlushMode
     private final WaitQueue fsyncComplete = newWaitQueue(); // TODO (expected): this is only used for testing, can we remove this?
+    private final MonotonicClock clock = preciseTime;
 
     // a signal and flag that callers outside the flusher thread can use
     // to signal they want the journal segments to be flushed to disk
     private final Semaphore haveWork = newSemaphore(1);
     private volatile boolean flushRequested;
 
-    private final FlushMethod<K, V> syncFlushMethod;
-    private final FlushMethod<K, V> asyncFlushMethod;
+    private final Mode<K, V> mode;
     private final Callbacks callbacks;
 
     Flusher(Journal<K, V> journal, Callbacks callbacks)
     {
         this.journal = journal;
         this.params = journal.params;
-        this.syncFlushMethod = syncFlushMethod(params);
-        this.asyncFlushMethod = asyncFlushMethod(params);
+        this.mode = mode(params);
         this.callbacks = callbacks;
     }
 
     void start()
     {
         String flushExecutorName = journal.name + "-disk-flusher-" + toLowerCaseLocalized(params.flushMode().toString());
-        flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new FlushRunnable(preciseTime), SAFE, NON_DAEMON, SYNCHRONIZED);
+        flushStartedAt = clock.now();
+        flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new FlushRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
     }
 
     void shutdown() throws InterruptedException
@@ -112,6 +112,7 @@ final class Flusher<K, V>
     }
 
     @Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
+    // waits for writes to complete before triggering an fsync
     private class FlushRunnable implements Interruptible.Task
     {
         @Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
@@ -182,6 +183,7 @@ final class Flusher<K, V>
                 fsyncStartedFor = startedAt;
                 // synchronized to prevent thread interrupts while performing IO operations and also
                 // clear interrupted status to prevent ClosedByInterruptException in ActiveSegment::flush
+                int fsyncedTo;
                 synchronized (this)
                 {
                     boolean ignore = Thread.interrupted();
@@ -191,17 +193,19 @@ final class Flusher<K, V>
                         journal.closeActiveSegmentAndOpenAsStatic(fsyncing);
                         fsyncing = journal.getActiveSegment(fsyncing.descriptor.timestamp + 1);
                     }
-                    fsyncing.fsync();
+                    fsyncedTo = fsyncTo.writtenToAtLeast();
+                    fsyncTo.fsync();
                 }
+                fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, startedAt);
                 fsyncFinishedFor = startedAt;
                 fsyncComplete.signalAll();
                 long finishedAt = clock.now();
                 processDuration(startedAt, finishedAt);
             }
 
-            void afterFlush(long startedAt, ActiveSegment<K, V> segment, int syncedOffset)
+            void afterFlush(long startedAt, ActiveSegment<K, V> segment)
             {
-                long requireFsyncTo = startedAt - periodicFlushLagBlockNanos();
+                long requireFsyncTo = startedAt - periodicBlockNanos();
 
                 fsyncUpTo = segment;
                 fsyncWaitingSince = startedAt;
@@ -210,26 +214,11 @@ final class Flusher<K, V>
 
                 if (requireFsyncTo > fsyncFinishedFor)
                     awaitFsyncAt(requireFsyncTo, journal.metrics.waitingOnFlush.time());
-                callbacks.onFlush(segment.descriptor.timestamp, syncedOffset);
-            }
-
-            private void doNoOpFlush(long startedAt)
-            {
-                if (fsyncFinishedFor >= fsyncWaitingSince)
-                {
-                    fsyncFinishedFor = startedAt;
-                }
-                else
-                {
-                    // if the flusher is still running, update the waitingSince register
-                    fsyncWaitingSince = startedAt;
-                    notify(awaitingWork);
-                }
+                callbacks.onFsync();
             }
         }
 
         private final NoSpamLogger noSpamLogger;
-        private final MonotonicClock clock;
         private final @Nullable FSyncRunnable fSyncRunnable;
 
         private ActiveSegment<K, V> current = null;
@@ -240,10 +229,9 @@ final class Flusher<K, V>
         private long duration = 0;              // time spent flushing since firstLaggedAt
         private long lagDuration = 0;                // cumulative lag since firstLaggedAt
 
-        FlushRunnable(MonotonicClock clock)
+        FlushRunnable()
         {
             this.noSpamLogger = NoSpamLogger.wrap(logger, 5, MINUTES);
-            this.clock = clock;
             this.fSyncRunnable = params.flushMode() == PERIODIC ? newFsyncRunnable() : null;
         }
 
@@ -287,6 +275,7 @@ final class Flusher<K, V>
 
             if (flushPeriodNanos <= 0)
             {
+                Invariants.checkState(params.flushMode() != PERIODIC);
                 haveWork.acquire(1);
             }
             else
@@ -296,7 +285,7 @@ final class Flusher<K, V>
             }
         }
 
-        private void doFlush(long startedAt) throws InterruptedException
+        private void doFlush(long startedAt)
         {
             boolean synchronousFsync = fSyncRunnable == null;
 
@@ -304,32 +293,33 @@ final class Flusher<K, V>
                 current = journal.oldestActiveSegment();
             ActiveSegment<K, V> newCurrent = journal.currentActiveSegment();
 
-            if (newCurrent == current && (newCurrent == null || !newCurrent.shouldFlush()))
-            {
-                if (synchronousFsync) fsyncFinishedFor = startedAt;
-                else fSyncRunnable.doNoOpFlush(startedAt);
-
-                if (current != null)
-                    callbacks.onFlush(current.descriptor.timestamp, (int) current.lastFlushedOffset());
+            if (newCurrent == null)
                 return;
-            }
-
-            Invariants.checkState(newCurrent != null);
 
             try
             {
                 while (current != newCurrent)
                 {
                     current.discardUnusedTail();
-                    current.flush(synchronousFsync);
+                    current.updateWrittenTo();
                     if (synchronousFsync)
+                    {
+                        current.fsync();
                         journal.closeActiveSegmentAndOpenAsStatic(current);
+                    }
                     current = journal.getActiveSegment(current.descriptor.timestamp + 1);
                 }
-                int syncedOffset = current.flush(synchronousFsync);
 
-                if (synchronousFsync) afterFSync(startedAt, current.descriptor.timestamp, syncedOffset);
-                else fSyncRunnable.afterFlush(startedAt, current, syncedOffset);
+                int writtenTo = current.updateWrittenTo();
+                if (synchronousFsync)
+                {
+                    current.fsync();
+                    afterFSync(startedAt, current.descriptor.timestamp, writtenTo);
+                }
+                else
+                {
+                    fSyncRunnable.afterFlush(startedAt, current);
+                }
             }
             catch (Throwable t)
             {
@@ -373,10 +363,11 @@ final class Flusher<K, V>
             }
         }
 
-        private void afterFSync(long startedAt, long syncedSegment, int syncedOffset)
+        private void afterFSync(long startedAt, long segment, int position)
         {
+            fsyncFinishedForPosition = new RecordPointer(segment, position, startedAt);
             fsyncFinishedFor = startedAt;
-            callbacks.onFlush(syncedSegment, syncedOffset);
+            callbacks.onFsync();
             fsyncComplete.signalAll();
             long finishedAt = clock.now();
             processDuration(startedAt, finishedAt);
@@ -390,88 +381,120 @@ final class Flusher<K, V>
         }
     }
 
-    @FunctionalInterface
-    private interface FlushMethod<K, V>
+    private interface Mode<K, V>
     {
-        void flush(ActiveSegment<K, V>.Allocation allocation);
+        void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc);
+        RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc);
+        boolean isDurable(RecordPointer recordPointer);
     }
 
-    private FlushMethod<K, V> syncFlushMethod(Params params)
+    private class BatchMode implements Mode<K, V>
     {
-        switch (params.flushMode())
-        {
-            default: throw new IllegalArgumentException();
-            case    BATCH: return this::waitForFlushBatch;
-            case    GROUP: return this::waitForFlushGroup;
-            case PERIODIC: return this::waitForFlushPeriodic;
-        }
-    }
-
-    private FlushMethod<K, V> asyncFlushMethod(Params params)
-    {
-        switch (params.flushMode())
-        {
-            default: throw new IllegalArgumentException();
-            case    BATCH: return this::asyncFlushBatch;
-            case    GROUP: return this::asyncFlushGroup;
-            case PERIODIC: return this::asyncFlushPeriodic;
-        }
-    }
-
-    void waitForFlush(ActiveSegment<K, V>.Allocation alloc)
-    {
-        syncFlushMethod.flush(alloc);
-    }
-
-    void asyncFlush(ActiveSegment<K, V>.Allocation alloc)
-    {
-        asyncFlushMethod.flush(alloc);
-    }
-
-    private void waitForFlushBatch(ActiveSegment<K, V>.Allocation alloc)
-    {
-        pending.incrementAndGet();
-        requestExtraFlush();
-        alloc.awaitFlush(journal.metrics.waitingOnFlush);
-        pending.decrementAndGet();
-        written.incrementAndGet();
-    }
-
-    private void waitForFlushGroup(ActiveSegment<K, V>.Allocation alloc)
-    {
-        pending.incrementAndGet();
-        alloc.awaitFlush(journal.metrics.waitingOnFlush);
-        pending.decrementAndGet();
-        written.incrementAndGet();
-    }
-
-    private void waitForFlushPeriodic(ActiveSegment<K, V>.Allocation ignore)
-    {
-        long expectedFlushTime = nanoTime() - periodicFlushLagBlockNanos();
-        if (fsyncFinishedFor < expectedFlushTime)
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
         {
             pending.incrementAndGet();
-            awaitFsyncAt(expectedFlushTime, journal.metrics.waitingOnFlush.time());
+            requestExtraFlush();
+            alloc.awaitDurable(journal.metrics.waitingOnFlush);
             pending.decrementAndGet();
+            written.incrementAndGet();
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            requestExtraFlush();
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer pointer)
+        {
+            return pointer.compareTo(fsyncFinishedForPosition) <= 0;
         }
-        written.incrementAndGet();
     }
 
-    private void asyncFlushBatch(ActiveSegment<K, V>.Allocation alloc)
+    private class GroupMode implements Mode<K, V>
     {
-        requestExtraFlush();
-        written.incrementAndGet();
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+        {
+            pending.incrementAndGet();
+            alloc.awaitDurable(journal.metrics.waitingOnFlush);
+            pending.decrementAndGet();
+            written.incrementAndGet();
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer pointer)
+        {
+            return pointer.compareTo(fsyncFinishedForPosition) <= 0;
+        }
     }
 
-    private void asyncFlushGroup(ActiveSegment<K, V>.Allocation alloc)
+    private class PeriodicMode implements Mode<K, V>
     {
-        written.incrementAndGet();
+        @Override
+        public void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+        {
+            RecordPointer pointer = flushAsync(alloc);
+
+            long expectedFsyncTime = pointer.writtenAt - periodicBlockNanos();
+            if (expectedFsyncTime > fsyncFinishedFor)
+            {
+                pending.incrementAndGet();
+                awaitFsyncAt(expectedFsyncTime, journal.metrics.waitingOnFlush.time());
+                pending.decrementAndGet();
+            }
+        }
+
+        @Override
+        public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
+        {
+            written.incrementAndGet();
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), clock.now());
+        }
+
+        @Override
+        public boolean isDurable(RecordPointer alloc)
+        {
+            long expectedFsyncTime = alloc.writtenAt - periodicBlockNanos();
+            return expectedFsyncTime <= fsyncFinishedFor;
+        }
     }
 
-    private void asyncFlushPeriodic(ActiveSegment<K, V>.Allocation ignore)
+    Mode<K, V> mode(Params params)
     {
-        requestExtraFlush();
-        written.incrementAndGet();
+        switch (params.flushMode())
+        {
+            default: throw new AssertionError("Unexpected FlushMode: " + params.flushMode());
+            case BATCH: return new BatchMode();
+            case GROUP: return new GroupMode();
+            case PERIODIC: return new PeriodicMode();
+        }
+    }
+
+    RecordPointer flush(ActiveSegment<K, V>.Allocation alloc)
+    {
+        return mode.flushAsync(alloc);
+    }
+
+    void flushAndAwaitDurable(ActiveSegment<K, V>.Allocation alloc)
+    {
+        mode.flushAndAwaitDurable(alloc);
+    }
+
+    boolean isDurable(RecordPointer pointer)
+    {
+        return mode.isDurable(pointer);
     }
 
     /**
@@ -505,12 +528,12 @@ final class Flusher<K, V>
 
     private long flushPeriodNanos()
     {
-        return 1_000_000L * params.flushPeriodMillis();
+        return params.flushPeriod(NANOSECONDS);
     }
 
-    private long periodicFlushLagBlockNanos()
+    private long periodicBlockNanos()
     {
-        return 1_000_000L * params.periodicFlushLagBlock();
+        return params.periodicBlockPeriod(NANOSECONDS);
     }
 
     long pendingEntries()
@@ -531,8 +554,9 @@ final class Flusher<K, V>
          * completed and also flushed.
          * callbacks for all entries earlier than (segment, position) have finished execution.
          */
-        void onFlush(long segment, int position);
+        void onFsync();
 
+        // TODO (required): tie this to specific allocations..
         void onFlushFailed(Throwable cause);
     }
 }
diff --git a/src/java/org/apache/cassandra/journal/InMemoryIndex.java b/src/java/org/apache/cassandra/journal/InMemoryIndex.java
index 77fd7352ee..baa56853cc 100644
--- a/src/java/org/apache/cassandra/journal/InMemoryIndex.java
+++ b/src/java/org/apache/cassandra/journal/InMemoryIndex.java
@@ -102,7 +102,10 @@ final class InMemoryIndex<K> extends Index<K>
     @Override
     public long[] lookUp(K id)
     {
-        return mayContainId(id) ? index.getOrDefault(id, EMPTY) : EMPTY;
+        K lastId = lastId();
+        if (lastId == null || keySupport.compare(id, lastId) <= 0)
+            return index.getOrDefault(id, EMPTY);
+        return EMPTY;
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/journal/Index.java b/src/java/org/apache/cassandra/journal/Index.java
index ac1e7c1d91..fba9b99f86 100644
--- a/src/java/org/apache/cassandra/journal/Index.java
+++ b/src/java/org/apache/cassandra/journal/Index.java
@@ -21,8 +21,6 @@ import javax.annotation.Nullable;
 
 import org.apache.cassandra.utils.Closeable;
 
-import static com.google.common.collect.Iterables.any;
-
 /**
  * Mapping of client supplied ids to in-segment offsets
  */
@@ -75,15 +73,7 @@ abstract class Index<K> implements Closeable
         K firstId = firstId();
         K lastId = lastId();
 
-        return null != firstId && null != lastId && keySupport.compare(id, firstId) >= 0 && keySupport.compare(id, lastId) <= 0;
-    }
-
-    /**
-     * @return whether any of the ids falls within lower/upper bounds of the index
-     */
-    boolean mayContainIds(Iterable<K> ids)
-    {
-        return any(ids, this::mayContainId);
+        return null != firstId && keySupport.compare(id, firstId) >= 0 && (null == lastId || keySupport.compare(id, lastId) <= 0);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/journal/Journal.java b/src/java/org/apache/cassandra/journal/Journal.java
index 34aff492ea..1b5dc82c47 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
 import com.codahale.metrics.Timer.Context;
-import org.agrona.collections.ObjectHashSet;
 import org.apache.cassandra.concurrent.Interruptible;
 import org.apache.cassandra.concurrent.Interruptible.TerminateException;
 import org.apache.cassandra.concurrent.SequentialExecutorPlus;
@@ -51,12 +50,12 @@ import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.PathUtils;
-import org.apache.cassandra.journal.Segments.ReferencedSegment;
 import org.apache.cassandra.journal.Segments.ReferencedSegments;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Crc;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Simulate;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.utils.concurrent.WaitQueue;
 import org.jctools.queues.MpscUnboundedArrayQueue;
 
@@ -119,8 +118,9 @@ public class Journal<K, V> implements Shutdownable
     private final WaitQueue allocatorThreadWaitQueue = newWaitQueue();
     private final BooleanSupplier allocatorThreadWaitCondition = () -> (availableSegment == null);
     private final FlusherCallbacks flusherCallbacks;
+    final OpOrder readOrder = new OpOrder();
 
-    SequentialExecutorPlus closer;
+    SequentialExecutorPlus closer, releaser;
 
     private class FlusherCallbacks implements Flusher.Callbacks
     {
@@ -128,17 +128,14 @@ public class Journal<K, V> implements Shutdownable
         private List<WaitingFor> drained = new ArrayList<>();
 
         @Override
-        public void onFlush(long segment, int position)
+        public void onFsync()
         {
-            // TODO (required): this seems to be a big source of allocations
             waitingFor.drain(drained::add);
             List<WaitingFor> remaining = new ArrayList<>();
             for (WaitingFor wait : drained)
             {
-                if (wait.segment == segment && wait.position <= position)
-                    wait.run();
-                else
-                    remaining.add(wait);
+                if (flusher.isDurable(wait)) wait.run();
+                else remaining.add(wait);
             }
             drained = remaining;
         }
@@ -151,13 +148,10 @@ public class Journal<K, V> implements Shutdownable
 
         private void submit(RecordPointer pointer, Runnable runnable)
         {
-            if (isFlushed(pointer))
+            if (flusher.isDurable(pointer))
                 runnable.run();
             else
-            {
-                waitingFor.add(new WaitingFor(pointer.segment, pointer.position, runnable));
-                flusher.requestExtraFlush();
-            }
+                waitingFor.add(new WaitingFor(pointer, runnable));
         }
     }
 
@@ -165,9 +159,9 @@ public class Journal<K, V> implements Shutdownable
     {
         private final Runnable onFlush;
 
-        public WaitingFor(long segment, int position, Runnable onFlush)
+        public WaitingFor(RecordPointer pointer, Runnable onFlush)
         {
-            super(segment, position);
+            super(pointer);
             this.onFlush = onFlush;
         }
 
@@ -197,19 +191,9 @@ public class Journal<K, V> implements Shutdownable
         this.compactor = new Compactor<>(this, segmentCompactor);
     }
 
-    public boolean isFlushed(RecordPointer recordPointer)
+    public void onDurable(RecordPointer recordPointer, Runnable runnable)
     {
-        Segment<K, V> current = currentSegment;
-        if (current.descriptor.timestamp == recordPointer.segment)
-            return current.isFlushed(recordPointer.position);
-
-        return segments.get().isFlushed(recordPointer);
-    }
-
-    public void onFlush(RecordPointer recordPointer, Runnable runnable)
-    {
-        if (isFlushed(recordPointer)) runnable.run();
-        else flusherCallbacks.submit(recordPointer, runnable);
+        flusherCallbacks.submit(recordPointer, runnable);
     }
 
     public void start()
@@ -230,12 +214,13 @@ public class Journal<K, V> implements Shutdownable
 
         segments.set(Segments.of(StaticSegment.open(descriptors, keySupport)));
         closer = executorFactory().sequential(name + "-closer");
+        releaser = executorFactory().sequential(name + "-releaser");
         allocator = executorFactory().infiniteLoop(name + "-allocator", new AllocateRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
         advanceSegment(null);
-        flusher.start();
-        compactor.start();
         Invariants.checkState(state.compareAndSet(State.INITIALIZING, State.NORMAL),
                               "Unexpected journal state after initialization", state);
+        flusher.start();
+        compactor.start();
     }
 
     @VisibleForTesting
@@ -272,13 +257,16 @@ public class Journal<K, V> implements Shutdownable
                                   "Unexpected journal state while trying to shut down", state);
             allocator.shutdown();
             wakeAllocator(); // Wake allocator to force it into shutdown
+            // TODO (expected): why are we awaitingTermination here when we have a separate method for it?
             allocator.awaitTermination(1, TimeUnit.MINUTES);
             segmentPrepared.signalAll(); // Wake up all threads waiting on the new segment
             compactor.shutdown();
             compactor.awaitTermination(1, TimeUnit.MINUTES);
             flusher.shutdown();
             closer.shutdown();
+            releaser.shutdown();
             closer.awaitTermination(1, TimeUnit.MINUTES);
+            releaser.awaitTermination(1, TimeUnit.MINUTES);
             closeAllSegments();
             metrics.deregister();
             Invariants.checkState(state.compareAndSet(State.SHUTDOWN, State.TERMINATED),
@@ -303,6 +291,7 @@ public class Journal<K, V> implements Shutdownable
         boolean r = true;
         r &= allocator.awaitTermination(timeout, units);
         r &= closer.awaitTermination(timeout, units);
+        r &= releaser.awaitTermination(timeout, units);
         return r;
     }
 
@@ -323,9 +312,9 @@ public class Journal<K, V> implements Shutdownable
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
 
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.allSorted(true))
+            for (Segment<K, V> segment : segments.get().allSorted(true))
             {
                 if (segment.readLast(id, holder))
                 {
@@ -347,9 +336,9 @@ public class Journal<K, V> implements Shutdownable
     public void readAll(K id, RecordConsumer<K> consumer)
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.allSorted(false))
+            for (Segment<K, V> segment : segments.get().allSorted(false))
                 segment.readAll(id, holder, consumer);
         }
     }
@@ -390,9 +379,9 @@ public class Journal<K, V> implements Shutdownable
     {
         EntrySerializer.EntryHolder<K> holder = new EntrySerializer.EntryHolder<>();
 
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.all())
+            for (Segment<K, V> segment : segments.get().all())
             {
                 long[] offsets = segment.index().lookUp(id);
                 for (long offsetAndSize : offsets)
@@ -436,40 +425,18 @@ public class Journal<K, V> implements Shutdownable
     @SuppressWarnings("unused")
     public boolean readLast(K id, RecordConsumer<K> consumer)
     {
-        try (ReferencedSegments<K, V> segments = selectAndReference(id))
+        try (OpOrder.Group group = readOrder.start())
         {
-            for (Segment<K, V> segment : segments.all())
+            for (Segment<K, V> segment : segments.get().allSorted(false))
+            {
+                if (!segment.index().mayContainId(id))
+                    continue;
+
                 if (segment.readLast(id, consumer))
                     return true;
-        }
-        return false;
-    }
-
-    /**
-     * Test for existence of entries with specified ids.
-     *
-     * @return subset of ids to test that have been found in the journal
-     */
-    @SuppressWarnings("unused")
-    public Set<K> test(Set<K> test)
-    {
-        Set<K> present = new ObjectHashSet<>(test.size() + 1, 0.9f);
-        try (ReferencedSegments<K, V> segments = selectAndReference(test))
-        {
-            for (Segment<K, V> segment : segments.all())
-            {
-                for (K id : test)
-                {
-                    if (segment.index().lookUpLast(id) != -1)
-                    {
-                        present.add(id);
-                        if (test.size() == present.size())
-                            return present;
-                    }
-                }
             }
         }
-        return present;
+        return false;
     }
 
     /**
@@ -488,7 +455,7 @@ public class Journal<K, V> implements Shutdownable
             valueSerializer.serialize(id, record, dob, params.userVersion());
             ActiveSegment<K, V>.Allocation alloc = allocate(dob.getLength(), hosts);
             alloc.writeInternal(id, dob.unsafeGetBufferAndFlip(), hosts);
-            flusher.waitForFlush(alloc);
+            flusher.flushAndAwaitDurable(alloc);
         }
         catch (IOException e)
         {
@@ -514,20 +481,18 @@ public class Journal<K, V> implements Shutdownable
 
     public RecordPointer asyncWrite(K id, Writer writer, Set<Integer> hosts)
     {
-        RecordPointer recordPointer;
         try (DataOutputBuffer dob = DataOutputBuffer.scratchBuffer.get())
         {
             writer.write(dob, params.userVersion());
             ActiveSegment<K, V>.Allocation alloc = allocate(dob.getLength(), hosts);
-            recordPointer = alloc.write(id, dob.unsafeGetBufferAndFlip(), hosts);
-            flusher.asyncFlush(alloc);
+            alloc.write(id, dob.unsafeGetBufferAndFlip(), hosts);
+            return flusher.flush(alloc);
         }
         catch (IOException e)
         {
             // exception during record serialization into the scratch buffer
             throw new RuntimeException(e);
         }
-        return recordPointer;
     }
 
     private ActiveSegment<K, V>.Allocation allocate(int entrySize, Set<Integer> hosts)
@@ -615,7 +580,7 @@ public class Journal<K, V> implements Shutdownable
             availableSegment = null;
         }
         if (next != null)
-            next.closeAndDiscard();
+            next.closeAndDiscard(this);
     }
 
     private class AllocateRunnable implements Interruptible.Task
@@ -715,52 +680,9 @@ public class Journal<K, V> implements Shutdownable
         for (Segment<K, V> segment : segments.all())
         {
             if (segment.isActive())
-                ((ActiveSegment<K, V>) segment).closeAndIfEmptyDiscard();
+                ((ActiveSegment<K, V>) segment).closeAndIfEmptyDiscard(this);
             else
-                segment.close();
-        }
-    }
-
-    /**
-     * Select segments that could potentially have any entry with the specified id and
-     * attempt to grab references to them all.
-     *
-     * @return a subset of segments with references to them
-     */
-    ReferencedSegments<K, V> selectAndReference(K id)
-    {
-        while (true)
-        {
-            ReferencedSegments<K, V> referenced = segments().selectAndReference(s -> s.index().mayContainId(id));
-            if (null != referenced)
-                return referenced;
-        }
-    }
-
-    /**
-     * Select segments that could potentially have any entry with the specified ids and
-     * attempt to grab references to them all.
-     *
-     * @return a subset of segments with references to them
-     */
-    ReferencedSegments<K, V> selectAndReference(Iterable<K> ids)
-    {
-        while (true)
-        {
-            ReferencedSegments<K, V> referenced = segments().selectAndReference(s -> s.index().mayContainIds(ids));
-            if (null != referenced)
-                return referenced;
-        }
-    }
-
-    @SuppressWarnings("unused")
-    ReferencedSegment<K, V> selectAndReference(long segmentTimestamp)
-    {
-        while (true)
-        {
-            ReferencedSegment<K, V> referenced = segments().selectAndReference(segmentTimestamp);
-            if (null != referenced)
-                return referenced;
+                segment.close(this);
         }
     }
 
@@ -886,10 +808,11 @@ public class Journal<K, V> implements Shutdownable
         public void run()
         {
             activeSegment.discardUnusedTail();
-            activeSegment.flush(true);
+            activeSegment.updateWrittenTo();
+            activeSegment.fsync();
             activeSegment.persistComponents();
             replaceCompletedSegment(activeSegment, StaticSegment.open(activeSegment.descriptor, keySupport));
-            activeSegment.release();
+            activeSegment.release(Journal.this);
         }
     }
 
@@ -898,7 +821,7 @@ public class Journal<K, V> implements Shutdownable
         if (activeSegment.isEmpty())
         {
             removeEmptySegment(activeSegment);
-            activeSegment.closeAndDiscard();
+            activeSegment.closeAndDiscard(this);
             return;
         }
 
@@ -1000,6 +923,7 @@ public class Journal<K, V> implements Shutdownable
      */
     public class StaticSegmentIterator implements Closeable
     {
+        // TODO (expected): use MergeIterator
         private final PriorityQueue<StaticSegment.KeyOrderReader<K>> readers;
         private final ReferencedSegments<K, V> segments;
 
diff --git a/src/java/org/apache/cassandra/journal/KeySupport.java b/src/java/org/apache/cassandra/journal/KeySupport.java
index 13cb902ddf..efc41aa6c8 100644
--- a/src/java/org/apache/cassandra/journal/KeySupport.java
+++ b/src/java/org/apache/cassandra/journal/KeySupport.java
@@ -36,10 +36,12 @@ public interface KeySupport<K> extends Comparator<K>
     int serializedSize(int userVersion);
 
     void serialize(K key, DataOutputPlus out, int userVersion) throws IOException;
+    void serialize(K key, ByteBuffer out, int userVersion) throws IOException;
 
     K deserialize(DataInputPlus in, int userVersion) throws IOException;
 
     K deserialize(ByteBuffer buffer, int position, int userVersion);
+    K deserialize(ByteBuffer buffer, int userVersion);
 
     void updateChecksum(Checksum crc, K key, int userVersion);
 
diff --git a/src/java/org/apache/cassandra/journal/Metadata.java b/src/java/org/apache/cassandra/journal/Metadata.java
index e8224ca64e..fcdbe1fba2 100644
--- a/src/java/org/apache/cassandra/journal/Metadata.java
+++ b/src/java/org/apache/cassandra/journal/Metadata.java
@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.journal;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
@@ -45,6 +46,7 @@ final class Metadata
     private final Set<Integer> unmodifiableHosts;
     private final Map<Integer, Integer> recordsPerHost;
 
+    private int fsyncLimit;
     private volatile int recordsCount;
     private static final AtomicIntegerFieldUpdater<Metadata> recordsCountUpdater =
         AtomicIntegerFieldUpdater.newUpdater(Metadata.class, "recordsCount");
@@ -73,6 +75,11 @@ final class Metadata
             recordsPerHost.compute(host, (k, v) -> null == v ? 1 : v + 1);
     }
 
+    int fsyncLimit()
+    {
+        return fsyncLimit;
+    }
+
     private void incrementRecordsCount()
     {
         recordsCountUpdater.incrementAndGet(this);
@@ -186,12 +193,12 @@ final class Metadata
         }
     }
 
-    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport)
     {
         Int2IntHashMap recordsPerHost = new Int2IntHashMap(Integer.MIN_VALUE);
         int recordsCount = 0;
 
-        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, fsyncedLimit))
+        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, Integer.MAX_VALUE))
         {
             while (reader.advance())
             {
@@ -203,13 +210,19 @@ final class Metadata
                 ++recordsCount;
             }
         }
+        catch (JournalReadError e)
+        {
+            // we expect EOF when rebuilding
+            if (!(e.getCause() instanceof EOFException))
+                throw e;
+        }
 
         return new Metadata(recordsPerHost, recordsCount);
     }
 
-    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport)
     {
-        Metadata metadata = rebuild(descriptor, keySupport, fsyncedLimit);
+        Metadata metadata = rebuild(descriptor, keySupport);
         metadata.persist(descriptor);
         return metadata;
     }
diff --git a/src/java/org/apache/cassandra/journal/Params.java b/src/java/org/apache/cassandra/journal/Params.java
index 56bacce1d9..452f024fbc 100644
--- a/src/java/org/apache/cassandra/journal/Params.java
+++ b/src/java/org/apache/cassandra/journal/Params.java
@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.concurrent.TimeUnit;
+
 public interface Params
 {
     enum FlushMode { BATCH, GROUP, PERIODIC }
@@ -40,23 +42,18 @@ public interface Params
 
     boolean enableCompaction();
 
-    int compactionPeriodMillis();
+    long compactionPeriod(TimeUnit units);
 
     /**
      * @return milliseconds between journal flushes
      */
-    int flushPeriodMillis();
-
-    default int flushPeriodNanos()
-    {
-        return flushPeriodMillis() * 1_000_000;
-    }
+    long flushPeriod(TimeUnit units);
 
     /**
-     * @return milliseconds to block writes for while waiting for a slow disk flush to complete
-     *         when in {@link FlushMode#PERIODIC} mode
+     * @return to block writes for while waiting for a slow fsync to complete
+     * when in {@link FlushMode#PERIODIC} mode
      */
-    int periodicFlushLagBlock();
+    long periodicBlockPeriod(TimeUnit units);
 
     /**
      * @return user provided version to use for key and value serialization
diff --git a/src/java/org/apache/cassandra/journal/RecordPointer.java b/src/java/org/apache/cassandra/journal/RecordPointer.java
index 2b3e8ea6b8..1439e63c90 100644
--- a/src/java/org/apache/cassandra/journal/RecordPointer.java
+++ b/src/java/org/apache/cassandra/journal/RecordPointer.java
@@ -26,11 +26,23 @@ public class RecordPointer implements Comparable<RecordPointer>
 {
     public final long segment; // unique segment id
     public final int position; // record start position within the segment
+    public final long writtenAt; // only set for periodic mode
 
     public RecordPointer(long segment, int position)
+    {
+        this(segment, position, 0);
+    }
+
+    public RecordPointer(long segment, int position, long writtenAt)
     {
         this.segment = segment;
         this.position = position;
+        this.writtenAt = writtenAt;
+    }
+
+    public RecordPointer(RecordPointer pointer)
+    {
+        this(pointer.segment, pointer.position, pointer.writtenAt);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/journal/Segment.java b/src/java/org/apache/cassandra/journal/Segment.java
index 77f7c68fea..6f8cde0358 100644
--- a/src/java/org/apache/cassandra/journal/Segment.java
+++ b/src/java/org/apache/cassandra/journal/Segment.java
@@ -20,25 +20,44 @@ package org.apache.cassandra.journal;
 import java.nio.ByteBuffer;
 
 import accord.utils.Invariants;
+import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.utils.*;
-import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.SelfRefCounted;
 
-public abstract class Segment<K, V> implements Closeable, RefCounted<Segment<K, V>>
+public abstract class Segment<K, V> implements SelfRefCounted<Segment<K, V>>, Comparable<Segment<K, V>>
 {
+    protected abstract static class Tidier implements Tidy, Runnable
+    {
+        OpOrder.Barrier await;
+        ExecutorPlus executor;
+
+        abstract void onUnreferenced();
+
+        public final void run()
+        {
+            await.await();
+            onUnreferenced();
+        }
+
+        public final void tidy()
+        {
+            executor.execute(this);
+        }
+    }
+
     final File file;
     final Descriptor descriptor;
-    final SyncedOffsets syncedOffsets;
     final Metadata metadata;
     final KeySupport<K> keySupport;
 
     ByteBuffer buffer;
 
-    Segment(Descriptor descriptor, SyncedOffsets syncedOffsets, Metadata metadata, KeySupport<K> keySupport)
+    Segment(Descriptor descriptor, Metadata metadata, KeySupport<K> keySupport)
     {
         this.file = descriptor.fileFor(Component.DATA);
         this.descriptor = descriptor;
-        this.syncedOffsets = syncedOffsets;
         this.metadata = metadata;
         this.keySupport = keySupport;
     }
@@ -98,7 +117,27 @@ public abstract class Segment<K, V> implements Closeable, RefCounted<Segment<K,
         }
     }
 
+    @Override
+    public int compareTo(Segment<K, V> that)
+    {
+        return this.descriptor.compareTo(that.descriptor);
+    }
+
     abstract boolean read(int offset, int size, EntrySerializer.EntryHolder<K> into);
 
-    abstract void release();
+    abstract void close(Journal<K, V> journal);
+
+    void release(Journal<K, V> journal)
+    {
+        Ref<Segment<K, V>> selfRef = selfRef();
+        Tidier tidier = (Tidier) selfRef.tidier();
+        if (journal != null)
+        {
+            // permitted to be null ONLY for tests
+            tidier.await = journal.readOrder.newBarrier();
+            tidier.await.issue();
+            tidier.executor = journal.releaser;
+        }
+        selfRef.release();
+    }
 }
diff --git a/src/java/org/apache/cassandra/journal/SegmentWriter.java b/src/java/org/apache/cassandra/journal/SegmentWriter.java
deleted file mode 100644
index 09797c363e..0000000000
--- a/src/java/org/apache/cassandra/journal/SegmentWriter.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.journal;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import com.google.common.primitives.Ints;
-
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.io.util.TrackedDataOutputPlus;
-
-final class SegmentWriter<K> implements Closeable
-{
-    private final Descriptor descriptor;
-    private final KeySupport<K> keySupport;
-
-    private final InMemoryIndex<K> index;
-    private final Metadata metadata;
-
-    private final File file;
-    private FileOutputStreamPlus untrackedOut;
-    private TrackedDataOutputPlus trackedOut;
-
-    private SegmentWriter(Descriptor descriptor, KeySupport<K> keySupport)
-    {
-        this.descriptor = descriptor;
-        this.keySupport = keySupport;
-
-        index = InMemoryIndex.create(keySupport);
-        metadata = Metadata.create();
-
-        file = descriptor.fileFor(Component.DATA);
-        try
-        {
-            untrackedOut = new FileOutputStreamPlus(file);
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-        trackedOut = TrackedDataOutputPlus.wrap(untrackedOut);
-    }
-
-    static <K> SegmentWriter<K> create(Descriptor descriptor, KeySupport<K> keySupport)
-    {
-        return new SegmentWriter<>(descriptor, keySupport);
-    }
-
-    int write(K key, ByteBuffer record, Set<Integer> hosts)
-    {
-        int position = position();
-        try
-        {
-            EntrySerializer.write(key, record, hosts, keySupport, trackedOut, descriptor.userVersion);
-            index.update(key, position, position() - position);
-            metadata.update(hosts);
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-        return position;
-    }
-
-    int position()
-    {
-        return Ints.checkedCast(trackedOut.position());
-    }
-
-    @Override
-    public void close()
-    {
-        try
-        {
-            untrackedOut.flush();
-            untrackedOut.sync();
-            untrackedOut.close();
-        }
-        catch (IOException e)
-        {
-            throw new JournalWriteError(descriptor, file, e);
-        }
-
-        try (SyncedOffsets syncedOffsets = SyncedOffsets.active(descriptor))
-        {
-            syncedOffsets.mark(position(), true);
-        }
-
-        index.persist(descriptor);
-        metadata.persist(descriptor);
-
-        untrackedOut = null;
-        trackedOut = null;
-    }
-}
diff --git a/src/java/org/apache/cassandra/journal/Segments.java b/src/java/org/apache/cassandra/journal/Segments.java
index cc98750fc4..6601d75549 100644
--- a/src/java/org/apache/cassandra/journal/Segments.java
+++ b/src/java/org/apache/cassandra/journal/Segments.java
@@ -17,15 +17,13 @@
  */
 package org.apache.cassandra.journal;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.List;
 import java.util.function.Predicate;
 
 import accord.utils.Invariants;
+import accord.utils.SortedArrays.SortedArrayList;
 import org.agrona.collections.Long2ObjectHashMap;
-import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.concurrent.Refs;
 
 /**
@@ -36,6 +34,7 @@ import org.apache.cassandra.utils.concurrent.Refs;
 class Segments<K, V>
 {
     private final Long2ObjectHashMap<Segment<K, V>> segments;
+    private SortedArrayList<Segment<K, V>> sorted;
 
     Segments(Long2ObjectHashMap<Segment<K, V>> segments)
     {
@@ -108,10 +107,9 @@ class Segments<K, V>
      */
     List<Segment<K, V>> allSorted(boolean asc)
     {
-        List<Segment<K, V>> segments = new ArrayList<>(this.segments.values());
-        if (asc) segments.sort(Comparator.comparing(s -> s.descriptor));
-        else segments.sort((o1, o2) -> -o1.descriptor.compareTo(o2.descriptor));
-        return segments;
+        if (sorted == null)
+            sorted = SortedArrayList.<Segment<K, V>>copyUnsorted(segments.values(), Segment[]::new);
+        return asc ? sorted : sorted.reverse();
     }
 
     void selectActive(long maxTimestamp, Collection<ActiveSegment<K, V>> into)
@@ -132,12 +130,14 @@ class Segments<K, V>
 
     ActiveSegment<K, V> oldestActive()
     {
-        Segment<K, V> oldest = null;
-        for (Segment<K, V> segment : segments.values())
-            if (segment.isActive() && (oldest == null || segment.descriptor.timestamp <= oldest.descriptor.timestamp))
-                oldest = segment;
-
-        return oldest == null ? null : oldest.asActive();
+        List<Segment<K, V>> sorted = allSorted(true);
+        for (int i = 0 ; i < sorted.size() ; ++i)
+        {
+            Segment<K, V> segment = sorted.get(i);
+            if (segment.isActive())
+                return segment.asActive();
+        }
+        return null;
     }
 
     Segment<K, V> get(long timestamp)
@@ -158,8 +158,26 @@ class Segments<K, V>
      *
      * @return a subset of segments with references to them, or {@code null} if failed to grab the refs
      */
-    @SuppressWarnings("resource")
     ReferencedSegments<K, V> selectAndReference(Predicate<Segment<K, V>> test)
+    {
+        Long2ObjectHashMap<Segment<K, V>> selectedSegments = select(test).segments;
+        Refs<Segment<K, V>> refs = null;
+        if (!selectedSegments.isEmpty())
+        {
+            refs = Refs.tryRef(selectedSegments.values());
+            if (null == refs)
+                return null;
+        }
+        return new ReferencedSegments<>(selectedSegments, refs);
+    }
+
+    /**
+     * Select segments that could potentially have an entry with the specified ids and
+     * attempt to grab references to them all.
+     *
+     * @return a subset of segments with references to them, or {@code null} if failed to grab the refs
+     */
+    Segments<K, V> select(Predicate<Segment<K, V>> test)
     {
         Long2ObjectHashMap<Segment<K, V>> selectedSegments = null;
         for (Segment<K, V> segment : segments.values())
@@ -175,14 +193,7 @@ class Segments<K, V>
         if (null == selectedSegments)
             selectedSegments = emptyMap();
 
-        Refs<Segment<K, V>> refs = null;
-        if (!selectedSegments.isEmpty())
-        {
-            refs = Refs.tryRef(selectedSegments.values());
-            if (null == refs)
-                return null;
-        }
-        return new ReferencedSegments<>(selectedSegments, refs);
+        return new Segments<>(selectedSegments);
     }
 
     static class ReferencedSegments<K, V> extends Segments<K, V> implements AutoCloseable
@@ -203,49 +214,6 @@ class Segments<K, V>
         }
     }
 
-    boolean isFlushed(RecordPointer recordPointer)
-    {
-        Segment<K, V> segment = segments.get(recordPointer.segment);
-        if (null == segment)
-            throw new IllegalArgumentException("Can not reference segment " + recordPointer.segment);
-        return segment.isFlushed(recordPointer.position);
-    }
-
-    ReferencedSegment<K, V> selectAndReference(long segmentTimestamp)
-    {
-        Segment<K, V> segment = segments.get(segmentTimestamp);
-        if (null == segment)
-            return new ReferencedSegment<>(null, null);
-        Ref<Segment<K, V>> ref = segment.tryRef();
-        if (null == ref)
-            return null;
-        return new ReferencedSegment<>(segment, ref);
-    }
-
-    static class ReferencedSegment<K, V> implements AutoCloseable
-    {
-        private final Segment<K, V> segment;
-        private final Ref<Segment<K, V>> ref;
-
-        ReferencedSegment(Segment<K, V> segment, Ref<Segment<K, V>> ref)
-        {
-            this.segment = segment;
-            this.ref = ref;
-        }
-
-        Segment<K, V> segment()
-        {
-            return segment;
-        }
-
-        @Override
-        public void close()
-        {
-            if (null != ref)
-                ref.release();
-        }
-    }
-
     private static final Long2ObjectHashMap<?> EMPTY_MAP = new Long2ObjectHashMap<>();
 
     @SuppressWarnings("unchecked")
@@ -256,6 +224,6 @@ class Segments<K, V>
 
     private static <K> Long2ObjectHashMap<K> newMap(int expectedSize)
     {
-        return new Long2ObjectHashMap<>(0, 0.65f, false);
+        return new Long2ObjectHashMap<>(expectedSize, 0.65f, false);
     }
 }
diff --git a/src/java/org/apache/cassandra/journal/StaticSegment.java b/src/java/org/apache/cassandra/journal/StaticSegment.java
index bf46ca8ecd..5f83a6eb1b 100644
--- a/src/java/org/apache/cassandra/journal/StaticSegment.java
+++ b/src/java/org/apache/cassandra/journal/StaticSegment.java
@@ -26,14 +26,12 @@ import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.locks.LockSupport;
 
-import accord.utils.Invariants;
 import org.agrona.collections.IntHashSet;
-import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.utils.Closeable;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
 
 /**
@@ -45,6 +43,7 @@ import org.apache.cassandra.utils.concurrent.Ref;
 public final class StaticSegment<K, V> extends Segment<K, V>
 {
     final FileChannel channel;
+    final int fsyncLimit;
 
     private final Ref<Segment<K, V>> selfRef;
 
@@ -53,15 +52,15 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     private StaticSegment(Descriptor descriptor,
                           FileChannel channel,
                           MappedByteBuffer buffer,
-                          SyncedOffsets syncedOffsets,
                           OnDiskIndex<K> index,
                           Metadata metadata,
                           KeySupport<K> keySupport)
     {
-        super(descriptor, syncedOffsets, metadata, keySupport);
+        super(descriptor, metadata, keySupport);
         this.index = index;
 
         this.channel = channel;
+        this.fsyncLimit = metadata.fsyncLimit();
         this.buffer = buffer;
 
         selfRef = new Ref<>(this, new Tidier<>(descriptor, channel, buffer, index));
@@ -93,21 +92,17 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         if (!Component.DATA.existsFor(descriptor))
             throw new IllegalArgumentException("Data file for segment " + descriptor + " doesn't exist");
 
-        SyncedOffsets syncedOffsets = Component.SYNCED_OFFSETS.existsFor(descriptor)
-                                    ? SyncedOffsets.load(descriptor)
-                                    : SyncedOffsets.absent();
-
         Metadata metadata = Component.METADATA.existsFor(descriptor)
                           ? Metadata.load(descriptor)
-                          : Metadata.rebuildAndPersist(descriptor, keySupport, syncedOffsets.syncedOffset());
+                          : Metadata.rebuildAndPersist(descriptor, keySupport);
 
         OnDiskIndex<K> index = Component.INDEX.existsFor(descriptor)
                              ? OnDiskIndex.open(descriptor, keySupport)
-                             : OnDiskIndex.rebuildAndPersist(descriptor, keySupport, syncedOffsets.syncedOffset());
+                             : OnDiskIndex.rebuildAndPersist(descriptor, keySupport, metadata.fsyncLimit());
 
         try
         {
-            return internalOpen(descriptor, syncedOffsets, index, metadata, keySupport);
+            return internalOpen(descriptor, index, metadata, keySupport);
         }
         catch (IOException e)
         {
@@ -116,55 +111,27 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     }
 
     private static <K, V> StaticSegment<K, V> internalOpen(
-        Descriptor descriptor, SyncedOffsets syncedOffsets, OnDiskIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
+        Descriptor descriptor, OnDiskIndex<K> index, Metadata metadata, KeySupport<K> keySupport)
     throws IOException
     {
         File file = descriptor.fileFor(Component.DATA);
         FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
         MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
-        return new StaticSegment<>(descriptor, channel, buffer, syncedOffsets, index, metadata, keySupport);
+        return new StaticSegment<>(descriptor, channel, buffer, index, metadata, keySupport);
     }
 
-    @Override
-    public void close()
+    public void close(Journal<K, V> journal)
     {
-        try
-        {
-            channel.close();
-        }
-        catch (IOException e)
-        {
-            throw new RuntimeException("Could not close static segment " + descriptor, e);
-        }
-
-        release();
+        release(journal);
     }
 
     /**
      * Waits until this segment is unreferenced, closes it, and deltes all files associated with it.
      */
-    void discard()
+    void discard(Journal<K, V> journal)
     {
-        // TODO: consider moving deletion logic to Tidier instead of busy-looping here
-        waitUntilUnreferenced();
-        close();
-        for (Component component : Component.values())
-        {
-            File file = descriptor.fileFor(component);
-            if (file.exists())
-                file.delete();
-        }
-    }
-
-    public void waitUntilUnreferenced()
-    {
-        while (true)
-        {
-            if (selfRef.globalCount() == 1)
-                return;
-
-            LockSupport.parkNanos(100);
-        }
+        ((Tidier)selfRef.tidier()).discard = true;
+        close(journal);
     }
 
     @Override
@@ -179,24 +146,25 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         return selfRef.ref();
     }
 
-    @Override
-    void release()
-    {
-        selfRef.release();
-    }
-
     @Override
     public String toString()
     {
         return "StaticSegment{" + descriptor + '}';
     }
 
-    private static final class Tidier<K> implements Tidy
+    @Override
+    public Ref<Segment<K, V>> selfRef()
+    {
+        return selfRef;
+    }
+
+    private static final class Tidier<K> extends Segment.Tidier implements Tidy
     {
         private final Descriptor descriptor;
         private final FileChannel channel;
         private final ByteBuffer buffer;
         private final Index<K> index;
+        boolean discard;
 
         Tidier(Descriptor descriptor, FileChannel channel, ByteBuffer buffer, Index<K> index)
         {
@@ -207,11 +175,21 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         }
 
         @Override
-        public void tidy()
+        void onUnreferenced()
         {
             FileUtils.clean(buffer);
             FileUtils.closeQuietly(channel);
             index.close();
+            if (discard)
+            {
+                Throwable fail = null;
+                for (Component component : Component.VALUES)
+                {
+                    try { descriptor.fileFor(component).deleteIfExists(); }
+                    catch (Throwable t) { fail = Throwables.merge(fail, t); }
+                }
+                Throwables.maybeFail(fail);
+            }
         }
 
         @Override
@@ -259,13 +237,9 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     boolean read(int offset, int size, EntrySerializer.EntryHolder<K> into)
     {
         ByteBuffer duplicate = buffer.duplicate().position(offset).limit(offset + size);
-        try (DataInputBuffer in = new DataInputBuffer(duplicate, false))
+        try
         {
-            if (!EntrySerializer.tryRead(into, keySupport, duplicate, in, syncedOffsets.syncedOffset(), descriptor.userVersion))
-                return false;
-
-            Invariants.checkState(in.available() == 0);
-            return true;
+            return 0 <= EntrySerializer.tryRead(into, keySupport, duplicate, fsyncLimit, descriptor.userVersion);
         }
         catch (IOException e)
         {
@@ -278,7 +252,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
      */
     void forEachRecord(RecordConsumer<K> consumer)
     {
-        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, syncedOffsets.syncedOffset()))
+        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, fsyncLimit))
         {
             while (reader.advance())
             {
@@ -389,13 +363,11 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     static final class SequentialReader<K> extends Reader<K>
     {
         private final int fsyncedLimit; // exclusive
-        private final DataInputBuffer in;
 
         SequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
         {
             super(descriptor, keySupport);
             this.fsyncedLimit = fsyncedLimit;
-            in = new DataInputBuffer(buffer, false);
         }
 
         @Override
@@ -413,8 +385,10 @@ public final class StaticSegment<K, V> extends Segment<K, V>
             offset = buffer.position();
             try
             {
-                if (!EntrySerializer.tryRead(holder, keySupport, buffer, in, fsyncedLimit, descriptor.userVersion))
+                int length = EntrySerializer.tryRead(holder, keySupport, buffer.duplicate(), fsyncedLimit, descriptor.userVersion);
+                if (length < 0)
                     return eof();
+                buffer.position(offset + length);
             }
             catch (IOException e)
             {
diff --git a/src/java/org/apache/cassandra/journal/SyncedOffsets.java b/src/java/org/apache/cassandra/journal/SyncedOffsets.java
deleted file mode 100644
index 8a89f72ad7..0000000000
--- a/src/java/org/apache/cassandra/journal/SyncedOffsets.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.journal;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.NoSuchFileException;
-import java.util.zip.CRC32;
-
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.io.util.FileOutputStreamPlus;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.Closeable;
-import org.apache.cassandra.utils.Crc;
-
-import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt;
-
-/**
- * Keeps track of fsynced limits of a data file. Enables us to treat invalid
- * records that are known to have been fsynced to disk differently from those
- * that aren't.
- * <p/>
- * On disk representation is a sequence of 2-int tuples of {synced offset, CRC32(synced offset)}
- */
-interface SyncedOffsets extends Closeable
-{
-    /**
-     * @return furthest known synced offset
-     */
-    int syncedOffset();
-
-    /**
-     * Record an offset as synced to disk.
-     *
-     * @param offset the offset into datafile, up to which contents have been fsynced (exclusive)
-     */
-    void mark(int offset, boolean fsync);
-
-    void fsync();
-
-    @Override
-    default void close()
-    {
-    }
-
-    /**
-     * @return a disk-backed synced offset tracker for a new {@link ActiveSegment}
-     */
-    static Active active(Descriptor descriptor)
-    {
-        return new Active(descriptor);
-    }
-
-    /**
-     * Load an existing log of synced offsets from disk into an immutable instance.
-     */
-    static Static load(Descriptor descriptor)
-    {
-        return Static.load(descriptor);
-    }
-
-    /**
-     * @return a placeholder instance in case this component is missing
-     */
-    static Absent absent()
-    {
-        return Absent.INSTANCE;
-    }
-
-    /**
-     * Single-threaded, file-based list of synced offsets.
-     */
-    final class Active implements SyncedOffsets
-    {
-        private final Descriptor descriptor;
-
-        private final FileOutputStreamPlus output;
-        private volatile int syncedOffset;
-
-        private Active(Descriptor descriptor)
-        {
-            this.descriptor = descriptor;
-
-            File file = descriptor.fileFor(Component.SYNCED_OFFSETS);
-            if (file.exists())
-                throw new IllegalArgumentException("Synced offsets file " + file + " already exists");
-
-            try
-            {
-                output = file.newOutputStream(File.WriteMode.OVERWRITE);
-            }
-            catch (UncheckedIOException | FSWriteError e)
-            {
-                // extract original cause and throw as JournalWriteError
-                throw new JournalWriteError(descriptor, file, e.getCause());
-            }
-            catch (NoSuchFileException e)
-            {
-                throw new AssertionError(); // unreachable
-            }
-        }
-
-        @Override
-        public int syncedOffset()
-        {
-            return syncedOffset;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            if (offset < syncedOffset)
-                throw new IllegalArgumentException("offset " + offset + " is smaller than previous mark " + offset);
-
-            CRC32 crc = Crc.crc32();
-            updateChecksumInt(crc, offset);
-
-            try
-            {
-                output.writeInt(offset);
-                output.writeInt((int) crc.getValue());
-                output.flush();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-
-            syncedOffset = offset;
-            if (fsync) fsync();
-        }
-
-        public void fsync()
-        {
-            try
-            {
-                output.sync();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-        }
-
-        @Override
-        public void close()
-        {
-            fsync();
-
-            try
-            {
-                output.close();
-            }
-            catch (IOException e)
-            {
-                throw new JournalWriteError(descriptor, Component.SYNCED_OFFSETS, e);
-            }
-        }
-    }
-
-    final class Static implements SyncedOffsets
-    {
-        private final int syncedOffset;
-
-        static Static load(Descriptor descriptor)
-        {
-            File file = descriptor.fileFor(Component.SYNCED_OFFSETS);
-            if (!file.exists())
-                throw new IllegalArgumentException("Synced offsets file " + file + " doesn't exist");
-
-            int syncedOffset = 0;
-            try (RandomAccessReader reader = RandomAccessReader.open(file))
-            {
-                CRC32 crc = Crc.crc32();
-                while (reader.bytesRemaining() >= 8)
-                {
-                    int offset = reader.readInt();
-                    updateChecksumInt(crc, offset);
-                    int readCrc = reader.readInt();
-                    if (readCrc != (int) crc.getValue())
-                        break;
-                    syncedOffset = offset;
-                    Crc.initialize(crc);
-                }
-            }
-            catch (Throwable t)
-            {
-                throw new JournalReadError(descriptor, file, t);
-            }
-
-            return new Static(syncedOffset);
-        }
-
-        Static(int offset)
-        {
-            this.syncedOffset = offset;
-        }
-
-        @Override
-        public int syncedOffset()
-        {
-            return syncedOffset;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void fsync()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    final class Absent implements SyncedOffsets
-    {
-        static final Absent INSTANCE = new Absent();
-
-        @Override
-        public int syncedOffset()
-        {
-            return 0;
-        }
-
-        @Override
-        public void mark(int offset, boolean fsync)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void fsync()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/metrics/AccordStateCacheMetrics.java b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java
similarity index 94%
rename from src/java/org/apache/cassandra/metrics/AccordStateCacheMetrics.java
rename to src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java
index b00793087e..1250c34751 100644
--- a/src/java/org/apache/cassandra/metrics/AccordStateCacheMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/AccordCacheMetrics.java
@@ -26,7 +26,7 @@ import com.codahale.metrics.Histogram;
 import static org.apache.cassandra.metrics.CacheMetrics.TYPE_NAME;
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
-public class AccordStateCacheMetrics extends CacheAccessMetrics
+public class AccordCacheMetrics extends CacheAccessMetrics
 {
     public static final String OBJECT_SIZE = "ObjectSize";
 
@@ -36,7 +36,7 @@ public class AccordStateCacheMetrics extends CacheAccessMetrics
 
     private final String scope;
 
-    public AccordStateCacheMetrics(String scope)
+    public AccordCacheMetrics(String scope)
     {
         super(new DefaultNameFactory(TYPE_NAME, scope));
         objectSize = Metrics.histogram(factory.createMetricName(OBJECT_SIZE), false);
diff --git a/src/java/org/apache/cassandra/metrics/AccordMetrics.java b/src/java/org/apache/cassandra/metrics/AccordMetrics.java
index 8064dee66a..8195d46f45 100644
--- a/src/java/org/apache/cassandra/metrics/AccordMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/AccordMetrics.java
@@ -49,6 +49,7 @@ public class AccordMetrics
     public static final String PROGRESS_LOG_SIZE = "ProgressLogSize";
 
     public static final String DEPENDENCIES = "Dependencies";
+    public static final String EPHEMERAL = "Ephemeral";
     public static final String FAST_PATHS = "FastPaths";
     public static final String SLOW_PATHS = "SlowPaths";
     public static final String PREEMPTS = "Preempts";
diff --git a/src/java/org/apache/cassandra/schema/TableId.java b/src/java/org/apache/cassandra/schema/TableId.java
index 5cdaf1c2b6..dbdf6fcc8d 100644
--- a/src/java/org/apache/cassandra/schema/TableId.java
+++ b/src/java/org/apache/cassandra/schema/TableId.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nullable;
 
+import org.agrona.collections.Hashing;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.commons.lang3.ArrayUtils;
@@ -41,6 +42,7 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.UUIDGen;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@@ -51,18 +53,24 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
  * This is essentially a UUID, but we wrap it as it's used quite a bit in the code and having a nicely named class make
  * the code more readable.
  */
-public class TableId implements Comparable<TableId>
+public final class TableId implements Comparable<TableId>
 {
     public static final long MAGIC = 1956074401491665062L;
     public static final long EMPTY_SIZE = ObjectSizes.measureDeep(new UUID(0, 0));
 
     private static final ConcurrentHashMap<TableId, TableId> internCache = new ConcurrentHashMap<>();
 
-    private final UUID id;
+    final long msb, lsb;
 
     private TableId(UUID id)
     {
-        this.id = id;
+        this(id.getMostSignificantBits(), id.getLeastSignificantBits());
+    }
+
+    private TableId(long msb, long lsb)
+    {
+        this.msb = msb;
+        this.lsb = lsb;
     }
 
     public static TableId fromUUID(UUID id)
@@ -70,6 +78,11 @@ public class TableId implements Comparable<TableId>
         return new TableId(id);
     }
 
+    public static TableId fromRaw(long msb, long lsb)
+    {
+        return new TableId(msb, lsb);
+    }
+
     // TODO: should we be using UUID.randomUUID()?
     public static TableId generate()
     {
@@ -145,43 +158,46 @@ public class TableId implements Comparable<TableId>
 
     public String toHexString()
     {
-        return ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(id));
+        return ByteBufferUtil.bytesToHex(ByteBuffer.wrap(UUIDGen.decompose(msb, lsb)));
     }
 
     public UUID asUUID()
     {
-        return id;
+        return new UUID(msb, lsb);
     }
 
     @Override
-    public final int hashCode()
+    public int hashCode()
     {
-        return id.hashCode();
+        return Hashing.hash(msb ^ lsb);
     }
 
     @Override
     public final boolean equals(Object o)
     {
-        return this == o || (o instanceof TableId && this.id.equals(((TableId) o).id));
+        if (o == this | o == null) return o == this;
+        if (o.getClass() != TableId.class) return false;
+        TableId that = (TableId) o;
+        return this.msb == that.msb && this.lsb == that.lsb;
     }
 
     @Override
     public String toString()
     {
-        return id.toString();
+        return new UUID(msb, lsb).toString();
     }
 
     public void serialize(DataOutput out) throws IOException
     {
-        out.writeLong(id.getMostSignificantBits());
-        out.writeLong(id.getLeastSignificantBits());
+        out.writeLong(msb);
+        out.writeLong(lsb);
     }
 
     public <V> int serialize(V dst, ValueAccessor<V> accessor, int offset)
     {
         int position = offset;
-        position += accessor.putLong(dst, position, id.getMostSignificantBits());
-        position += accessor.putLong(dst, position, id.getLeastSignificantBits());
+        position += accessor.putLong(dst, position, msb);
+        position += accessor.putLong(dst, position, lsb);
         return position - offset;
     }
 
@@ -197,12 +213,12 @@ public class TableId implements Comparable<TableId>
 
     public static TableId deserialize(DataInput in) throws IOException
     {
-        return new TableId(new UUID(in.readLong(), in.readLong()));
+        return new TableId(in.readLong(), in.readLong());
     }
 
     public static <V> TableId deserialize(V src, ValueAccessor<V> accessor, int offset)
     {
-        return new TableId(new UUID(accessor.getLong(src, offset), accessor.getLong(src, offset + TypeSizes.LONG_SIZE)));
+        return new TableId(accessor.getLong(src, offset), accessor.getLong(src, offset + TypeSizes.LONG_SIZE));
     }
 
     public TableId intern()
@@ -212,9 +228,10 @@ public class TableId implements Comparable<TableId>
     }
 
     @Override
-    public int compareTo(TableId o)
+    public int compareTo(TableId that)
     {
-        return id.compareTo(o.id);
+        int c = Long.compare(this.msb, that.msb);
+        return c != 0 ? c : Long.compare(this.lsb, that.lsb);
     }
 
     public static final IVersionedSerializer<TableId> serializer = new IVersionedSerializer<>()
@@ -263,5 +280,4 @@ public class TableId implements Comparable<TableId>
     {
         ScheduledExecutors.scheduledFastTasks.scheduleSelfRecurring(internCache::clear, 1, TimeUnit.HOURS);
     }
-
 }
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index cfbbd96e63..555937dd96 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -179,6 +179,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.concat;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordEphemeralReadEnabledEnabled;
 import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
@@ -205,6 +206,7 @@ import static org.apache.cassandra.service.consensus.migration.ConsensusMigratio
 import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.splitMutationsIntoAccordAndNormal;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
+import static org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode.none;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
 import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
@@ -2199,7 +2201,7 @@ public class StorageProxy implements StorageProxyMBean
         consistencyLevel = consistencyLevelForAccordRead(cm, group, consistencyLevel);
         TxnRead read = TxnRead.createSerialRead(group.queries, consistencyLevel);
         Txn.Kind kind = Read;
-        if (transactionalMode == TransactionalMode.full && DatabaseDescriptor.getAccordEphemeralReadEnabledEnabled() && group.queries.size() == 1)
+        if (transactionalMode == TransactionalMode.full && getAccordEphemeralReadEnabledEnabled() && group.queries.size() == 1 && group.metadata().params.transactionalMigrationFrom == none)
             kind = EphemeralRead;
         Txn txn = new Txn.InMemory(kind, read.keys(), read, TxnQuery.ALL, null);
         AsyncTxnResult asyncTxnResult = AccordService.instance().coordinateAsync(txn, consistencyLevel, requestTime);
diff --git a/src/java/org/apache/cassandra/service/accord/AccordCache.java b/src/java/org/apache/cassandra/service/accord/AccordCache.java
new file mode 100644
index 0000000000..f504b61739
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordCache.java
@@ -0,0 +1,1290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.ToLongFunction;
+import java.util.stream.Stream;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.api.RoutingKey;
+import accord.local.Command;
+import accord.local.cfk.CommandsForKey;
+import accord.primitives.Routable;
+import accord.primitives.SaveStatus;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.IntrusiveLinkedList;
+import accord.utils.Invariants;
+import accord.utils.QuadFunction;
+import accord.utils.TriFunction;
+import accord.utils.async.Cancellable;
+import org.agrona.collections.Object2ObjectHashMap;
+import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.exceptions.UnknownTableException;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.metrics.CacheAccessMetrics;
+import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
+import org.apache.cassandra.service.accord.events.CacheEvents;
+import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.NoSpamLogger.NoSpamLogStatement;
+import org.apache.cassandra.utils.ObjectSizes;
+
+import static accord.utils.Invariants.checkState;
+import static org.apache.cassandra.net.MessagingService.current_version;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED;
+
+/**
+ * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types.
+ * </p>
+ * Supports dynamic object sizes. After each acquire/free cycle, the cacheable objects size is recomputed to
+ * account for data added/removed during txn processing if it's modified flag is set
+ *
+ * TODO (required): we only iterate over unreferenced entries
+ */
+public class AccordCache implements CacheSize
+{
+    private static final Logger logger = LoggerFactory.getLogger(AccordCache.class);
+    private static final NoSpamLogStatement evictNoEvict = NoSpamLogger.getStatement(logger, "Found and expired {} marked no evict, with age {}, exceeding its expected max age of {}", 1L, TimeUnit.MINUTES);
+
+    // Debug mode to verify that loading from journal + system tables results in
+    // functionally identical (or superceding) command to the one we've just evicted.
+    private static boolean VALIDATE_LOAD_ON_EVICT = false;
+
+    @VisibleForTesting
+    public static void validateLoadOnEvict(boolean value)
+    {
+        VALIDATE_LOAD_ON_EVICT = value;
+    }
+
+    public interface Adapter<K, V, S>
+    {
+        @Nullable V load(AccordCommandStore commandStore, K key);
+        @Nullable Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk);
+        // a result of null means we can immediately evict, without saving
+        @Nullable V quickShrink(V value);
+        // a result of null means we cannot shrink, and should save/evict as appropriate
+        @Nullable Object fullShrink(K key, V value);
+        @Nullable V inflate(K key, Object shrunk);
+        long estimateHeapSize(V value);
+        long estimateShrunkHeapSize(Object shrunk);
+        boolean validate(AccordCommandStore commandStore, K key, V value);
+        S safeRef(AccordCacheEntry<K, V> node);
+
+        default AccordCacheEntry<K, V> newEntry(K key, AccordCache.Type<K, V, ?>.Instance owner)
+        {
+            return AccordCacheEntry.createReadyToLoad(key, owner);
+        }
+    }
+
+    static class Stats
+    {
+        long queries;
+        long hits;
+        long misses;
+    }
+
+    public static final class ImmutableStats
+    {
+        public final long queries;
+        public final long hits;
+        public final long misses;
+        
+        public ImmutableStats(Stats stats)
+        {
+            queries = stats.queries;
+            hits = stats.hits;
+            misses = stats.misses;
+        }
+    }
+
+    private final List<Type<?, ?, ?>> types = new CopyOnWriteArrayList<>();
+    private final Function<Runnable, Cancellable> saveExecutor;
+    private final AccordCacheEntry.OnSaved onSaved;
+    // TODO (required): monitor this queue and periodically clean up entries, or implement an eviction deadline system
+    private final IntrusiveLinkedList<AccordCacheEntry<?,?>> evictQueue = new IntrusiveLinkedList<>();
+    private final IntrusiveLinkedList<AccordCacheEntry<?,?>> noEvictQueue = new IntrusiveLinkedList<>();
+
+    private int unreferencedBytes;
+    private int unreferenced;
+    private long maxSizeInBytes;
+    private long bytesCached;
+    private int noEvictGeneration;
+    private boolean shrinkingOn = true;
+
+    @VisibleForTesting
+    final AccordCacheMetrics metrics;
+    final Stats stats = new Stats();
+
+    public AccordCache(Function<Runnable, Cancellable> saveExecutor, AccordCacheEntry.OnSaved onSaved, long maxSizeInBytes, AccordCacheMetrics metrics)
+    {
+        this.saveExecutor = saveExecutor;
+        this.onSaved = onSaved;
+        this.maxSizeInBytes = maxSizeInBytes;
+        this.metrics = metrics;
+    }
+
+    @Override
+    public void setCapacity(long sizeInBytes)
+    {
+        maxSizeInBytes = sizeInBytes;
+        maybeShrinkOrEvictSomeNodes();
+    }
+
+    public void setShrinkingOn(boolean shrinkingOn)
+    {
+        this.shrinkingOn = shrinkingOn;
+    }
+
+    @Override
+    public long capacity()
+    {
+        return maxSizeInBytes;
+    }
+
+    /**
+     * Make sure we don't have any items lingering too long in the no evict queue, to avoid cache memory leaks
+     */
+    void processNoEvictQueue()
+    {
+        noEvictGeneration = (noEvictGeneration + 1) & 0xffff;
+        if (noEvictQueue.isEmpty())
+            return;
+
+        Iterator<AccordCacheEntry<?, ?>> iter = noEvictQueue.iterator();
+        int skipCount = 3;
+        while (skipCount > 0 && iter.hasNext())
+        {
+            AccordCacheEntry<?, ?> entry = iter.next();
+            int age = (noEvictGeneration - entry.noEvictGeneration()) & 0xffff;
+            if (age >= entry.noEvictMaxAge())
+            {
+                evictNoEvict.warn(entry, age, entry.noEvictMaxAge());
+                evict(entry, true);
+            }
+            else
+            {
+                --skipCount;
+            }
+        }
+    }
+
+    /*
+     * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around
+     * for longer to maximise the chances of hitting system tables fewer times (or not at all).
+     */
+    private void maybeShrinkOrEvictSomeNodes()
+    {
+        while (bytesCached > maxSizeInBytes && !evictQueue.isEmpty())
+        {
+            AccordCacheEntry<?, ?> node = evictQueue.peek();
+            shrinkOrEvict(node);
+        }
+    }
+
+    @VisibleForTesting
+    private <K, V> void shrinkOrEvict(AccordCacheEntry<K, V> node)
+    {
+        checkState(node.references() == 0);
+
+        if (shrinkingOn && node.tryShrink())
+        {
+            IntrusiveLinkedList<AccordCacheEntry<?,?>> queue;
+            queue = node.isNoEvict() ? noEvictQueue : evictQueue;
+            node.unlink();
+            queue.addLast(node);
+        }
+        else
+        {
+            tryEvict(node);
+        }
+    }
+
+    @VisibleForTesting
+    public <K, V> void tryEvict(AccordCacheEntry<K, V> node)
+    {
+        checkState(node.references() == 0);
+
+        if (node.isNoEvict())
+        {
+            node.unlink();
+            noEvictQueue.addLast(node);
+            return;
+        }
+
+        Status status = node.status();
+        switch (status)
+        {
+            default: throw new IllegalStateException("Unhandled status " + status);
+            case LOADING:
+                node.loading().loading.cancel();
+            case WAITING_TO_LOAD:
+                Invariants.paranoid(node.loadingOrWaiting().waiters == null);
+            case LOADED:
+                node.unlink();
+                evict(node, true);
+                break;
+            case MODIFIED:
+                Type<K, V, ?> parent = node.owner.parent();
+                node.save(saveExecutor, parent.adapter, onSaved);
+                boolean evict = node.status() == LOADED;
+                node.unlink();
+                if (evict) evict(node, true);
+        }
+    }
+
+    private void evict(AccordCacheEntry<?, ?> node, boolean updateUnreferenced)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace("Evicting {}", node);
+
+        checkState(node.isUnqueued());
+
+        if (updateUnreferenced)
+        {
+            unreferencedBytes -= node.sizeOnHeap;
+            --unreferenced;
+        }
+        bytesCached -= node.sizeOnHeap;
+        Type<?, ?, ?>.Instance owner = node.owner;
+        Type<?, ?, ?> parent = owner.parent();
+        parent.bytesCached -= node.sizeOnHeap;
+        --parent.size;
+
+        // TODO (expected): use listeners
+        if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT)
+            owner.validateLoadEvicted(node);
+
+        AccordCacheEntry<?, ?> self = node.owner.cache.remove(node.key());
+        Invariants.checkState(self.references() == 0);
+        checkState(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self);
+        node.notifyListeners(Listener::onEvict);
+        node.evicted();
+    }
+
+    <P, K, V> Collection<AccordTask<?>> load(BiFunction<P, Runnable, Cancellable> loadExecutor, P param, AccordCacheEntry<K, V> node, AccordCacheEntry.OnLoaded onLoaded)
+    {
+        Type<K, V, ?> parent = node.owner.parent();
+        return node.load(loadExecutor, param, parent.adapter, onLoaded).waiters();
+    }
+
+    <K, V> void loaded(AccordCacheEntry<K, V> node, V value)
+    {
+        node.loaded(value);
+        node.notifyListeners(Listener::onUpdate);
+    }
+
+    <K, V> void failedToLoad(AccordCacheEntry<K, V> node)
+    {
+        Invariants.checkState(node.references() == 0);
+        if (node.isUnqueued())
+        {
+            Invariants.checkState(node.status() == EVICTED);
+            return;
+        }
+        node.unlink();
+        node.failedToLoad();
+        evict(node, true);
+    }
+
+    <K, V> void saved(AccordCacheEntry<K, V> node, Object identity, Throwable fail)
+    {
+        if (node.saved(identity, fail) && node.references() == 0)
+            evictQueue.addFirst(node); // add to front since we have just saved, so we were eligible for eviction
+    }
+
+    public <K, V, S extends AccordSafeState<K, V>> void release(S safeRef, AccordTask<?> owner)
+    {
+        safeRef.global().owner.release(safeRef, owner);
+    }
+
+    public ImmutableStats stats()
+    {
+        return new ImmutableStats(stats);
+    }
+
+    public <K, V, S extends AccordSafeState<K, V>> Type<K, V, S> newType(Class<K> keyClass, Adapter<K, V, S> adapter)
+    {
+        Type<K, V, S> instance = new Type<>(keyClass, adapter);
+        types.add(instance);
+        return instance;
+    }
+
+    public <K, V, S extends AccordSafeState<K, V>> Type<K, V, S> newType(
+        Class<K> keyClass,
+        BiFunction<AccordCommandStore, K, V> loadFunction,
+        QuadFunction<AccordCommandStore, K, V, Object, Runnable> saveFunction,
+        Function<V, V> quickShrink,
+        TriFunction<AccordCommandStore, K, V, Boolean> validateFunction,
+        ToLongFunction<V> heapEstimator,
+        Function<AccordCacheEntry<K, V>, S> safeRefFactory)
+    {
+        return newType(keyClass, loadFunction, saveFunction, quickShrink, (i, j) -> j, (i, j) -> (V)j, validateFunction, heapEstimator, i -> 0, safeRefFactory);
+    }
+
+    public <K, V, S extends AccordSafeState<K, V>> Type<K, V, S> newType(
+        Class<K> keyClass,
+        BiFunction<AccordCommandStore, K, V> loadFunction,
+        QuadFunction<AccordCommandStore, K, V, Object, Runnable> saveFunction,
+        Function<V, V> quickShrink,
+        BiFunction<K, V, Object> fullShrink,
+        BiFunction<K, Object, V> inflate,
+        TriFunction<AccordCommandStore, K, V, Boolean> validateFunction,
+        ToLongFunction<V> heapEstimator,
+        ToLongFunction<Object> shrunkHeapEstimator,
+        Function<AccordCacheEntry<K, V>, S> safeRefFactory)
+    {
+        return newType(keyClass, new FunctionalAdapter<>(loadFunction, saveFunction, quickShrink,
+                                                         fullShrink, inflate,
+                                                         validateFunction, heapEstimator, shrunkHeapEstimator,
+                                                         safeRefFactory, AccordCacheEntry::createReadyToLoad));
+    }
+
+    public Collection<Type<?, ? ,? >> types()
+    {
+        return types;
+    }
+
+    public interface Listener<K, V>
+    {
+        default void onAdd(AccordCacheEntry<K, V> state) {}
+        default void onUpdate(AccordCacheEntry<K, V> state) {}
+        default void onEvict(AccordCacheEntry<K, V> state) {}
+    }
+
+    public class Type<K, V, S extends AccordSafeState<K, V>> implements CacheSize
+    {
+        public class Instance implements Iterable<AccordCacheEntry<K, V>>
+        {
+            final AccordCommandStore commandStore;
+            // TODO (desired): don't need to store key separately as stored in node; ideally use a hash set that allows us to get the current entry
+            private final Map<K, AccordCacheEntry<K, V>> cache = new Object2ObjectHashMap<>();
+            private List<Listener<K, V>> listeners = null;
+
+            public Instance(AccordCommandStore commandStore)
+            {
+                this.commandStore = commandStore;
+            }
+
+            public S acquire(K key)
+            {
+                AccordCacheEntry<K, V> node = acquire(key, false);
+                return adapter.safeRef(node);
+            }
+
+            public S acquireIfLoaded(K key)
+            {
+                AccordCacheEntry<K, V> node = acquire(key, true);
+                if (node == null)
+                    return null;
+                return adapter.safeRef(node);
+            }
+
+            public S acquire(AccordCacheEntry<K, V> node)
+            {
+                Invariants.checkState(node.owner == this);
+                acquireExisting(node, false);
+                return adapter.safeRef(node);
+            }
+
+            public void recordPreAcquired(AccordSafeState<K, V> ref)
+            {
+                Invariants.checkState(ref.global().owner == this);
+                incrementCacheHits();
+            }
+
+            private AccordCacheEntry<K, V> acquire(K key, boolean onlyIfLoaded)
+            {
+                incrementCacheQueries();
+                @SuppressWarnings("unchecked")
+                AccordCacheEntry<K, V> node = cache.get(key);
+                return node == null
+                       ? acquireAbsent(key, onlyIfLoaded)
+                       : acquireExisting(node, onlyIfLoaded);
+            }
+
+            /*
+             * Can only return a LOADING Node (or null)
+             */
+            private AccordCacheEntry<K, V> acquireAbsent(K key, boolean onlyIfLoaded)
+            {
+                incrementCacheMisses();
+                if (onlyIfLoaded)
+                    return null;
+                AccordCacheEntry<K, V> node = adapter.newEntry(key, this);
+                node.increment();
+
+                Object prev = cache.put(key, node);
+                node.initSize(parent());
+                Invariants.checkState(prev == null, "%s not absent from cache: %s already present", key, node);
+                ++size;
+                node.notifyListeners(Listener::onAdd);
+                maybeShrinkOrEvictSomeNodes();
+                return node;
+            }
+
+            /*
+             * Can't return EVICTED or INITIALIZED
+             */
+            private AccordCacheEntry<K, V> acquireExisting(AccordCacheEntry<K, V> node, boolean onlyIfLoaded)
+            {
+                boolean isLoaded = node.isLoaded();
+                if (isLoaded)
+                    incrementCacheHits();
+                else
+                    incrementCacheMisses();
+
+                if (onlyIfLoaded && !isLoaded)
+                    return null;
+
+                if (node.increment() == 1)
+                {
+                    --unreferenced;
+                    unreferencedBytes -= node.sizeOnHeap;
+                    node.unlink();
+                }
+
+                return node;
+            }
+
+            public void release(AccordSafeState<K, V> safeRef, AccordTask<?> owner)
+            {
+                K key = safeRef.global().key();
+                logger.trace("Releasing resources for {}: {}", key, safeRef);
+
+                AccordCacheEntry<K, V> node = cache.get(key);
+
+                checkState(!safeRef.invalidated());
+                checkState(safeRef.global() != null, "safeRef node is null for %s", key);
+                checkState(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node);
+                checkState(node.references() > 0, "references (%d) are zero for %s (%s)", node.references(), key, node);
+                checkState(node.isUnqueued());
+
+                boolean evict = false;
+                if (safeRef.hasUpdate())
+                {
+                    V update = safeRef.current();
+                    if (update != null)
+                        update = adapter.quickShrink(update);
+                    node.setExclusive(update);
+                    if (update == null)
+                    {
+                        if (node.is(MODIFIED))
+                            node.saved();
+                        evict = true;
+                    }
+                    node.notifyListeners(Listener::onUpdate);
+                }
+                else if (node.isLoadingOrWaiting())
+                {
+                    node.loadingOrWaiting().remove(owner);
+                }
+                else
+                {
+                    evict = node.is(LOADED) && node.isNull();
+                }
+                safeRef.invalidate();
+
+                if (node.decrement() == 0)
+                {
+                    if (evict)
+                    {
+                        evict(node, false);
+                        return;
+                    }
+
+                    ++unreferenced;
+                    unreferencedBytes += node.sizeOnHeap;
+                    Status status = node.status(); // status() completes
+                    switch (status)
+                    {
+                        default: throw new IllegalStateException("Unhandled status " + status);
+                        case WAITING_TO_LOAD:
+                        case LOADING:
+                        case LOADED:
+                        case MODIFIED:
+                            logger.trace("Moving {} with status {} to eviction queue", key, status);
+                            evictQueue.addLast(node);
+
+                        case SAVING:
+                        case FAILED_TO_SAVE:
+                            break; // can never evict, so no point in adding to eviction queue either
+                    }
+                }
+
+                maybeShrinkOrEvictSomeNodes();
+            }
+
+            public Stream<AccordCacheEntry<K, V>> stream()
+            {
+                return cache.values().stream();
+            }
+
+            Type<K, V, S> parent()
+            {
+                return Type.this;
+            }
+
+            @Override
+            public Iterator<AccordCacheEntry<K, V>> iterator()
+            {
+                return stream().iterator();
+            }
+
+            void validateLoadEvicted(AccordCacheEntry<?, ?> node)
+            {
+                @SuppressWarnings("unchecked")
+                AccordCacheEntry<K, V> state = (AccordCacheEntry<K, V>) node;
+                K key = state.key();
+                V evicted = state.tryGetFull();
+                if (evicted == null)
+                {
+                    try
+                    {
+                        Object shrunk = state.tryGetShrunk();
+                        if (shrunk != null)
+                            evicted = adapter.inflate(key, shrunk);
+                    }
+                    catch (RuntimeException rte)
+                    {
+                        if (rte.getCause() instanceof UnknownTableException)
+                            return;
+                        throw rte;
+                    }
+                }
+                if (!adapter.validate(node.owner.commandStore, key, evicted))
+                    throw new IllegalStateException("Reloaded value for key " + key + " is not equal to or fuller than evicted value " + evicted);
+            }
+
+            @VisibleForTesting
+            public AccordCacheEntry<K, V> getUnsafe(K key)
+            {
+                return cache.get(key);
+            }
+
+            public Set<K> keySet()
+            {
+                return cache.keySet();
+            }
+
+            @VisibleForTesting
+            public boolean isReferenced(K key)
+            {
+                AccordCacheEntry<K, V> node = cache.get(key);
+                return node != null && node.references() > 0;
+            }
+
+            @VisibleForTesting
+            boolean keyIsReferenced(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
+            {
+                AccordCacheEntry<?, ?> node = cache.get(key);
+                return node != null && node.references() > 0;
+            }
+
+            @VisibleForTesting
+            boolean keyIsCached(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
+            {
+                AccordCacheEntry<?, ?> node = cache.get(key);
+                return node != null;
+            }
+
+            @VisibleForTesting
+            int references(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
+            {
+                AccordCacheEntry<?, ?> node = cache.get(key);
+                return node != null ? node.references() : 0;
+            }
+
+            void notifyListeners(BiConsumer<Listener<K, V>, AccordCacheEntry<K, V>> notify, AccordCacheEntry<K, V> node)
+            {
+                notifyListeners(listeners, notify, node);
+                notifyListeners(typeListeners, notify, node);
+            }
+
+            void notifyListeners(List<Listener<K, V>> listeners, BiConsumer<Listener<K, V>, AccordCacheEntry<K, V>> notify, AccordCacheEntry<K, V> node)
+            {
+                if (listeners != null)
+                {
+                    for (int i = 0, size = listeners.size() ; i < size ; ++i)
+                        notify.accept(listeners.get(i), node);
+
+                }
+            }
+
+            public void register(Listener<K, V> l)
+            {
+                if (listeners == null)
+                    listeners = new ArrayList<>();
+                listeners.add(l);
+            }
+
+            public void unregister(Listener<K, V> l)
+            {
+                if (!tryUnregister(l))
+                    throw new AssertionError("Listener was not registered");
+            }
+
+            public boolean tryUnregister(Listener<K, V> l)
+            {
+                if (listeners == null || !listeners.remove(l))
+                    return false;
+                if (listeners.isEmpty())
+                    listeners = null;
+                return true;
+            }
+
+        }
+
+        private final Class<K> keyClass;
+        private Adapter<K, V, S> adapter;
+        private long bytesCached;
+        private int size;
+
+        @VisibleForTesting
+        final CacheAccessMetrics typeMetrics;
+        private final Stats stats = new Stats();
+        private List<Listener<K, V>> typeListeners = null;
+
+        public Type(
+            Class<K> keyClass,
+            Adapter<K, V, S> adapter)
+        {
+            this.keyClass = keyClass;
+            this.adapter = adapter;
+            this.typeMetrics = metrics.forInstance(keyClass);
+        }
+
+        void updateSize(long newSize, long delta, boolean isUnreferenced, boolean updateHistogram)
+        {
+            // TODO (expected): deprecate this in favour of a histogram snapshot of any point in time
+            bytesCached += delta;
+            AccordCache.this.bytesCached += delta;
+            if (updateHistogram) metrics.objectSize.update(newSize);
+            if (isUnreferenced) AccordCache.this.unreferencedBytes += delta;
+        }
+
+        // can be safely garbage collected if empty
+        Instance newInstance(AccordCommandStore commandStore)
+        {
+            return new Instance(commandStore);
+        }
+
+        private void incrementCacheQueries()
+        {
+            typeMetrics.requests.mark();
+            metrics.requests.mark();
+            stats.queries++;
+            AccordCache.this.stats.queries++;
+        }
+
+        private void incrementCacheHits()
+        {
+            typeMetrics.hits.mark();
+            metrics.hits.mark();
+            stats.hits++;
+            AccordCache.this.stats.hits++;
+        }
+
+        private void incrementCacheMisses()
+        {
+            typeMetrics.misses.mark();
+            metrics.misses.mark();
+            stats.misses++;
+            AccordCache.this.stats.misses++;
+        }
+
+        AccordCache parent()
+        {
+            return AccordCache.this;
+        }
+
+        public Stats stats()
+        {
+            return stats;
+        }
+
+        public ImmutableStats statsSnapshot()
+        {
+            return new ImmutableStats(stats);
+        }
+
+        public Stats globalStats()
+        {
+            return AccordCache.this.stats;
+        }
+
+        @VisibleForTesting
+        public void unsafeSetLoadFunction(BiFunction<AccordCommandStore, K, V> loadFunction)
+        {
+            if (adapter.getClass() != SettableWrapper.class)
+                adapter = new SettableWrapper<>(adapter);
+            ((SettableWrapper<K, V, S>)adapter).load = loadFunction;
+        }
+
+        public BiFunction<AccordCommandStore, K, V> unsafeGetLoadFunction()
+        {
+            if (adapter.getClass() != SettableWrapper.class)
+                adapter = new SettableWrapper<>(adapter);
+            return ((SettableWrapper<K, V, S>)adapter).load;
+        }
+
+        Adapter<K, V, S> adapter()
+        {
+            return adapter;
+        }
+
+        @Override
+        public long capacity()
+        {
+            return AccordCache.this.capacity();
+        }
+
+        @Override
+        public void setCapacity(long capacity)
+        {
+            throw new UnsupportedOperationException("Capacity is shared between all instances. Please set the capacity on the global cache");
+        }
+
+        @Override
+        public int size()
+        {
+            return size;
+        }
+
+        @Override
+        public long weightedSize()
+        {
+            return bytesCached;
+        }
+
+        public long globalAllocated()
+        {
+            return AccordCache.this.bytesCached;
+        }
+
+        public int globalReferencedEntries()
+        {
+            return AccordCache.this.numReferencedEntries();
+        }
+
+        public int globalUnreferencedEntries()
+        {
+            return AccordCache.this.numUnreferencedEntries();
+        }
+
+        public void register(Listener<K, V> l)
+        {
+            if (typeListeners == null)
+                typeListeners = new ArrayList<>();
+            typeListeners.add(l);
+        }
+
+        public void unregister(Listener<K, V> l)
+        {
+            if (typeListeners == null)
+                throw new AssertionError("No listeners exist");
+            if (!typeListeners.remove(l))
+                throw new AssertionError("Listener was not registered");
+            if (typeListeners.isEmpty())
+                typeListeners = null;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Instance{" +
+                   ", keyClass=" + keyClass +
+                   '}';
+        }
+    }
+
+    @VisibleForTesting
+    AccordCacheEntry<?, ?> head()
+    {
+        Iterator<AccordCacheEntry<?, ?>> iter = evictQueue.iterator();
+        return iter.hasNext() ? iter.next() : null;
+    }
+
+    @VisibleForTesting
+    AccordCacheEntry<?, ?> tail()
+    {
+        AccordCacheEntry<?,?> last = null;
+        Iterator<AccordCacheEntry<?, ?>> iter = evictQueue.iterator();
+        while (iter.hasNext())
+            last = iter.next();
+        return last;
+    }
+
+    public boolean isEmpty()
+    {
+        return size() == 0;
+    }
+
+    Iterable<AccordCacheEntry<?, ?>> evictionQueue()
+    {
+        return evictQueue::iterator;
+    }
+
+    private int cacheSize()
+    {
+        int size = 0;
+        for (Type<?, ?, ?> type : types)
+            size += type.size();
+        return size;
+    }
+
+    @VisibleForTesting
+    int numReferencedEntries()
+    {
+        return cacheSize() - unreferenced;
+    }
+
+    @VisibleForTesting
+    int numUnreferencedEntries()
+    {
+        return unreferenced;
+    }
+
+    @VisibleForTesting
+    int unreferencedBytes()
+    {
+        return unreferencedBytes;
+    }
+
+    @Override
+    public int size()
+    {
+        return cacheSize();
+    }
+
+    @Override
+    public long weightedSize()
+    {
+        return bytesCached;
+    }
+
+    static <K, V> void registerJfrListener(int shardId, AccordCache.Type<K, V, ?> type, String name)
+    {
+        if (!DatabaseDescriptor.getAccordStateCacheListenerJFREnabled())
+            return;
+
+        type.register(new AccordCache.Listener<>() {
+            private final IdentityHashMap<AccordCacheEntry<?, ?>, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>();
+
+            @Override
+            public void onAdd(AccordCacheEntry<K, V> state)
+            {
+                CacheEvents.Add add = new CacheEvents.Add();
+                CacheEvents.Evict evict = new CacheEvents.Evict();
+                if (!add.isEnabled())
+                    return;
+                add.begin();
+                evict.begin();
+                add.shard = evict.shard = shardId;
+                add.instance = evict.instance = name;
+                add.key = evict.key = state.key().toString();
+                updateMutable(type, state, add);
+                add.commit();
+                pendingEvicts.put(state, evict);
+            }
+
+            @Override
+            public void onEvict(AccordCacheEntry<K, V> state)
+            {
+                CacheEvents.Evict event = pendingEvicts.remove(state);
+                if (event == null) return;
+                updateMutable(type, state, event);
+                event.commit();
+            }
+        });
+    }
+
+    private static void updateMutable(AccordCache.Type<?, ?, ?> type, AccordCacheEntry<?, ?> state, CacheEvents event)
+    {
+        event.status = state.status().name();
+
+        event.lastQueriedEstimatedSizeOnHeap = state.sizeOnHeap();
+
+        event.instanceAllocated = type.weightedSize();
+        AccordCache.Stats stats = type.stats();
+        event.instanceStatsQueries = stats.queries;
+        event.instanceStatsHits = stats.hits;
+        event.instanceStatsMisses = stats.misses;
+
+        event.globalSize = type.size();
+        event.globalReferenced = type.globalReferencedEntries();
+        event.globalUnreferenced = type.globalUnreferencedEntries();
+        event.globalCapacity = type.capacity();
+        event.globalAllocated = type.globalAllocated();
+
+        stats = type.globalStats();
+        event.globalStatsQueries = stats.queries;
+        event.globalStatsHits = stats.hits;
+        event.globalStatsMisses = stats.misses;
+
+        event.update();
+    }
+
+    static class FunctionalAdapter<K, V, S> implements Adapter<K, V, S>
+    {
+        final BiFunction<AccordCommandStore, K, V> load;
+        final QuadFunction<AccordCommandStore, K, V, Object, Runnable> save;
+        final Function<V, V> quickShrink;
+        final BiFunction<K, V, Object> shrink;
+        final BiFunction<K, Object, V> inflate;
+        final TriFunction<AccordCommandStore, K, V, Boolean> validate;
+        final ToLongFunction<V> estimateHeapSize;
+        final ToLongFunction<Object> estimateShrunkHeapSize;
+        final Function<AccordCacheEntry<K, V>, S> newSafeRef;
+        final BiFunction<K, AccordCache.Type<K, V, ?>.Instance, AccordCacheEntry<K, V>> newNode;
+
+        FunctionalAdapter(BiFunction<AccordCommandStore, K, V> load,
+                          QuadFunction<AccordCommandStore, K, V, Object, Runnable> save,
+                          Function<V, V> quickShrink, BiFunction<K, V, Object> shrink,
+                          BiFunction<K, Object, V> inflate,
+                          TriFunction<AccordCommandStore, K, V, Boolean> validate,
+                          ToLongFunction<V> estimateHeapSize,
+                          ToLongFunction<Object> estimateShrunkHeapSize,
+                          Function<AccordCacheEntry<K, V>, S> newSafeRef,
+                          BiFunction<K, Type<K, V, ?>.Instance, AccordCacheEntry<K, V>> newNode)
+        {
+            this.load = load;
+            this.save = save;
+            this.shrink = shrink;
+            this.quickShrink = quickShrink;
+            this.inflate = inflate;
+            this.validate = validate;
+            this.estimateHeapSize = estimateHeapSize;
+            this.estimateShrunkHeapSize = estimateShrunkHeapSize;
+            this.newSafeRef = newSafeRef;
+            this.newNode = newNode;
+        }
+
+        FunctionalAdapter(Adapter<K, V, S> wrap)
+        {
+            this(wrap::load, wrap::save, wrap::quickShrink, wrap::fullShrink, wrap::inflate, wrap::validate, wrap::estimateHeapSize, wrap::estimateShrunkHeapSize, wrap::safeRef, wrap::newEntry);
+        }
+
+        @Override
+        public V load(AccordCommandStore commandStore, K key)
+        {
+            return load.apply(commandStore, key);
+        }
+
+        @Override
+        public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk)
+        {
+            return save.apply(commandStore, key, value, shrunk);
+        }
+
+        @Override
+        public V quickShrink(V value)
+        {
+            return quickShrink.apply(value);
+        }
+
+        @Override
+        public Object fullShrink(K key, V value)
+        {
+            return shrink.apply(key, value);
+        }
+
+        @Override
+        public V inflate(K key, Object shrunk)
+        {
+            return inflate.apply(key, shrunk);
+        }
+
+        @Override
+        public boolean validate(AccordCommandStore commandStore, K key, V value)
+        {
+            return validate.apply(commandStore, key, value);
+        }
+
+        @Override
+        public long estimateHeapSize(V value)
+        {
+            return estimateHeapSize.applyAsLong(value);
+        }
+
+        @Override
+        public long estimateShrunkHeapSize(Object shrunk)
+        {
+            return estimateShrunkHeapSize.applyAsLong(shrunk);
+        }
+
+        @Override
+        public S safeRef(AccordCacheEntry<K, V> node)
+        {
+            return newSafeRef.apply(node);
+        }
+
+        @Override
+        public AccordCacheEntry<K, V> newEntry(K key, Type<K, V, ?>.Instance owner)
+        {
+            return newNode.apply(key, owner);
+        }
+    }
+
+    static class SettableWrapper<K, V, S> extends FunctionalAdapter<K, V, S>
+    {
+        volatile BiFunction<AccordCommandStore, K, V> load;
+
+        SettableWrapper(Adapter<K, V, S> wrapper)
+        {
+            super(wrapper);
+            this.load = super.load;
+        }
+
+        public static <K, V> Adapter<K, V, ?> loadOnly(BiFunction<AccordCommandStore, K, V> load)
+        {
+            SettableWrapper<K, V, ?> result = new SettableWrapper<>(new NoOpAdapter<>());
+            result.load = load;
+            return result;
+        }
+
+        @Override
+        public V load(AccordCommandStore commandStore, K key)
+        {
+            return load.apply(commandStore, key);
+        }
+    }
+
+    static class NoOpAdapter<K, V, S> implements Adapter<K, V, S>
+    {
+        @Override public V load(AccordCommandStore commandStore, K key) { return null; }
+        @Override public Runnable save(AccordCommandStore commandStore, K key, @Nullable V value, @Nullable Object shrunk) { return null; }
+        @Override public V quickShrink(V value) { return null; }
+        @Override public Object fullShrink(K key, V value) { return null; }
+        @Override public V inflate(K key, Object shrunk) { return null; }
+        @Override public long estimateHeapSize(V value) { return 0; }
+        @Override public long estimateShrunkHeapSize(Object shrunk) { return 0; }
+        @Override public boolean validate(AccordCommandStore commandStore, K key, V value) { return false; }
+        @Override public S safeRef(AccordCacheEntry<K, V> node) { return null; }
+    }
+
+    public static class CommandsForKeyAdapter implements Adapter<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>
+    {
+        public static final CommandsForKeyAdapter CFK_ADAPTER = new CommandsForKeyAdapter();
+        private CommandsForKeyAdapter() {}
+
+        @Override
+        public CommandsForKey load(AccordCommandStore commandStore, RoutingKey key)
+        {
+            return commandStore.loadCommandsForKey(key);
+        }
+
+        @Override
+        public Runnable save(AccordCommandStore commandStore, RoutingKey key, @Nullable CommandsForKey value, @Nullable Object serialized)
+        {
+            return commandStore.saveCommandsForKey(key, value, serialized);
+        }
+
+        @Override
+        public CommandsForKey quickShrink(CommandsForKey value)
+        {
+            return value;
+        }
+
+        @Override
+        public Object fullShrink(RoutingKey key, CommandsForKey value)
+        {
+            if (value.isEmpty())
+                return null;
+
+            return CommandsForKeySerializer.toBytesWithoutKey(value.maximalPrune());
+        }
+
+        @Override
+        public CommandsForKey inflate(RoutingKey key, Object shrunk)
+        {
+            return CommandsForKeySerializer.fromBytes(key, (ByteBuffer)shrunk);
+        }
+
+        @Override
+        public long estimateHeapSize(CommandsForKey value)
+        {
+            return AccordObjectSizes.commandsForKey(value);
+        }
+
+        @Override
+        public long estimateShrunkHeapSize(Object shrunk)
+        {
+            return ObjectSizes.sizeOnHeapOf((ByteBuffer) shrunk);
+        }
+
+        @Override
+        public boolean validate(AccordCommandStore commandStore, RoutingKey key, CommandsForKey value)
+        {
+            return commandStore.validateCommandsForKey(key, value);
+        }
+
+        @Override
+        public AccordSafeCommandsForKey safeRef(AccordCacheEntry<RoutingKey, CommandsForKey> node)
+        {
+            return new AccordSafeCommandsForKey(node);
+        }
+    }
+
+    public static class CommandAdapter implements Adapter<TxnId, Command, AccordSafeCommand>
+    {
+        public static final CommandAdapter COMMAND_ADAPTER = new CommandAdapter();
+        private CommandAdapter() {}
+
+        @Override
+        public Command load(AccordCommandStore commandStore, TxnId txnId)
+        {
+            Invariants.checkState(!txnId.is(Txn.Kind.EphemeralRead));
+            return commandStore.loadCommand(txnId);
+        }
+
+        @Override
+        public Runnable save(AccordCommandStore commandStore, TxnId txnId, @Nullable Command value, @Nullable Object serialized)
+        {
+            if (txnId.is(Routable.Domain.Key))
+                return null;
+
+            if (value == null)
+            {
+                value = inflate(txnId, serialized);
+                if (value == null)
+                    return null;
+            }
+
+            return commandStore.appendToKeyspace(txnId, value);
+        }
+
+        @Override
+        public Command quickShrink(Command value)
+        {
+            if (value.saveStatus() == SaveStatus.Uninitialised)
+                return null;
+            if (value.txnId().is(Txn.Kind.EphemeralRead) && value.saveStatus().compareTo(SaveStatus.ReadyToExecute) >= 0)
+                return null; // TODO (expected): should we manage this with the waiting callback? more work, but maybe cleaner/clearer/safer
+            return AccordCommandStore.prepareToCache(value);
+        }
+
+        @Override
+        public Object fullShrink(TxnId txnId, Command value)
+        {
+            if (txnId.is(Txn.Kind.EphemeralRead))
+                Invariants.checkState(value.saveStatus().compareTo(SaveStatus.ReadyToExecute) < 0);
+
+            try
+            {
+                return SavedCommand.asSerializedDiff(null, value, current_version);
+            }
+            catch (IOException e)
+            {
+                logger.warn("Failed to serialize {}", value, e);
+                return null;
+            }
+        }
+
+        @Override
+        public @Nullable Command inflate(TxnId key, Object serialized)
+        {
+            SavedCommand.Builder builder = new SavedCommand.Builder(key);
+            ByteBuffer buffer = (ByteBuffer) serialized;
+            buffer.mark();
+            try (DataInputBuffer buf = new DataInputBuffer(buffer, false))
+            {
+                builder.deserializeNext(buf, current_version);
+                return builder.construct();
+            }
+            catch (UnknownTableException e)
+            {
+                // TODO (required): log, and make sure callers correctly handle null
+                return null;
+            }
+            catch (IOException e)
+            {
+                // TODO (required): test and make sure recover safely from exceptions OR log and return null
+                throw new RuntimeException(e);
+            }
+            finally
+            {
+                buffer.reset();
+            }
+        }
+
+        @Override
+        public long estimateHeapSize(Command value)
+        {
+            return AccordObjectSizes.command(value);
+        }
+
+        @Override
+        public long estimateShrunkHeapSize(Object shrunk)
+        {
+            return ObjectSizes.sizeOnHeapOf((ByteBuffer) shrunk);
+        }
+
+        @Override
+        public boolean validate(AccordCommandStore commandStore, TxnId key, Command value)
+        {
+            return commandStore.validateCommand(key, value);
+        }
+
+        @Override
+        public AccordSafeCommand safeRef(AccordCacheEntry<TxnId, Command> node)
+        {
+            return new AccordSafeCommand(node);
+        }
+
+        @Override
+        public AccordCacheEntry<TxnId, Command> newEntry(TxnId txnId, Type<TxnId, Command, ?>.Instance owner)
+        {
+            AccordCacheEntry<TxnId, Command> node = new AccordCacheEntry<>(txnId, owner);
+            if (txnId.is(Txn.Kind.EphemeralRead))
+            {
+                node.initialize(null);
+                int maxAge = (int)Math.min(0xff, 1 + DatabaseDescriptor.getReadRpcTimeout(TimeUnit.SECONDS));
+                node.markNoEvict(owner.parent().parent().noEvictGeneration, maxAge);
+            }
+            else
+            {
+                node.readyToLoad();
+            }
+            return node;
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java
new file mode 100644
index 0000000000..011c1a37ba
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordCacheEntry.java
@@ -0,0 +1,664 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
+
+import accord.utils.ArrayBuffers.BufferList;
+import accord.utils.IntrusiveLinkedListNode;
+import accord.utils.Invariants;
+import accord.utils.async.Cancellable;
+import org.apache.cassandra.service.accord.AccordCache.Adapter;
+import org.apache.cassandra.utils.ObjectSizes;
+
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_LOAD;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.FAILED_TO_SAVE;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.LOADING;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.MODIFIED;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.SAVING;
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.WAITING_TO_LOAD;
+
+/**
+ * Global (per CommandStore) state of a cached entity (Command or CommandsForKey).
+ */
+public class AccordCacheEntry<K, V> extends IntrusiveLinkedListNode
+{
+    public enum Status
+    {
+        UNINITIALIZED,
+        WAITING_TO_LOAD(UNINITIALIZED),
+        LOADING(WAITING_TO_LOAD),
+        /**
+         * Consumers should never see this state
+         */
+        FAILED_TO_LOAD(LOADING),
+
+        LOADED(true, false, UNINITIALIZED, LOADING),
+        MODIFIED(true, false, LOADED),
+        SAVING(true, true, MODIFIED),
+
+        /**
+         * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization,
+         * or commit log has been stopped.
+         */
+        FAILED_TO_SAVE(true, true, SAVING),
+
+        UNUSED, // spacing to permit easier bit masks
+
+        EVICTED(WAITING_TO_LOAD, LOADING, LOADED, FAILED_TO_LOAD),
+        ;
+
+        static final Status[] VALUES = values();
+        static
+        {
+            MODIFIED.permittedFrom |= 1 << MODIFIED.ordinal();
+            MODIFIED.permittedFrom |= 1 << SAVING.ordinal();
+            MODIFIED.permittedFrom |= 1 << FAILED_TO_SAVE.ordinal();
+            LOADED.permittedFrom |= 1 << SAVING.ordinal();
+            LOADED.permittedFrom |= 1 << MODIFIED.ordinal();
+            for (Status status : VALUES)
+            {
+                Invariants.checkState((status.ordinal() & IS_LOADED) != 0 == status.loaded);
+                Invariants.checkState(((status.ordinal() & IS_LOADED) != 0 && (status.ordinal() & IS_NESTED) != 0) == status.nested);
+            }
+        }
+
+        final boolean loaded;
+        final boolean nested;
+        int permittedFrom;
+
+        Status(Status ... statuses)
+        {
+            this(false, false, statuses);
+        }
+
+        Status(boolean loaded, boolean nested, Status ... statuses)
+        {
+            this.loaded = loaded;
+            this.nested = nested;
+            for (Status status : statuses)
+                permittedFrom |= 1 << status.ordinal();
+        }
+    }
+
+    static final int STATUS_MASK = 0x0000001F;
+    static final int SHRUNK = 0x00000040;
+    static final int NO_EVICT = 0x00000020;
+    static final int IS_LOADED = 0x4;
+    static final int IS_NESTED = 0x2; // only valid to test if already tested NORMAL
+    static final int IS_LOADING_OR_WAITING_MASK = 0x6; // only valid to test if already tested NORMAL
+    static final int IS_LOADING_OR_WAITING = 0x2; // only valid to test if already tested NORMAL
+    static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCacheEntry<>(null, null));
+
+    private final K key;
+    final AccordCache.Type<K, V, ?>.Instance owner;
+
+    private Object state;
+    private int status;
+    int sizeOnHeap;
+    private volatile int references;
+    private static final AtomicIntegerFieldUpdater<AccordCacheEntry> referencesUpdater = AtomicIntegerFieldUpdater.newUpdater(AccordCacheEntry.class, "references");
+
+    AccordCacheEntry(K key, AccordCache.Type<K, V, ?>.Instance owner)
+    {
+        this.key = key;
+        this.owner = owner;
+    }
+
+    void unlink()
+    {
+        remove();
+    }
+
+    boolean isUnqueued()
+    {
+        return isFree();
+    }
+
+    public K key()
+    {
+        return key;
+    }
+
+    public int references()
+    {
+        return references;
+    }
+
+    public int increment()
+    {
+        return referencesUpdater.incrementAndGet(this);
+    }
+
+    public int decrement()
+    {
+        return referencesUpdater.decrementAndGet(this);
+    }
+
+    boolean isLoaded()
+    {
+        return (status & IS_LOADED) != 0;
+    }
+
+    boolean isNested()
+    {
+        Invariants.checkState(isLoaded());
+        return (status & IS_NESTED) != 0;
+    }
+
+    boolean isShrunk()
+    {
+        return (status & SHRUNK) != 0;
+    }
+
+    public boolean is(Status status)
+    {
+        return (this.status & STATUS_MASK) == status.ordinal();
+    }
+
+    boolean isLoadingOrWaiting()
+    {
+        return (status & IS_LOADING_OR_WAITING_MASK) == IS_LOADING_OR_WAITING;
+    }
+
+    public boolean isComplete()
+    {
+        return !is(LOADING) && !is(SAVING);
+    }
+
+    int noEvictGeneration()
+    {
+        Invariants.checkState(isNoEvict());
+        return (status >>> 8) & 0xffff;
+    }
+
+    int noEvictMaxAge()
+    {
+        Invariants.checkState(isNoEvict());
+        return status >>> 24;
+    }
+
+    boolean isNoEvict()
+    {
+        return (status & NO_EVICT) != 0;
+    }
+
+    int sizeOnHeap()
+    {
+        return sizeOnHeap;
+    }
+
+    void updateSize(AccordCache.Type<K, V, ?> parent)
+    {
+        // TODO (expected): we aren't weighing the keys
+        int newSizeOnHeap = Ints.saturatedCast(EMPTY_SIZE + estimateOnHeapSize(parent.adapter()));
+        parent.updateSize(newSizeOnHeap, newSizeOnHeap - sizeOnHeap, references == 0, true);
+        sizeOnHeap = newSizeOnHeap;
+    }
+
+    void initSize(AccordCache.Type<K, V, ?> parent)
+    {
+        // TODO (expected): we aren't weighing the keys
+        sizeOnHeap = Ints.saturatedCast(EMPTY_SIZE);
+        parent.updateSize(sizeOnHeap, sizeOnHeap, false, false);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Node{" + status() +
+               ", key=" + key() +
+               ", references=" + references +
+               "}@" + Integer.toHexString(System.identityHashCode(this));
+    }
+
+    public Status status()
+    {
+        return Status.VALUES[(status & STATUS_MASK)];
+    }
+
+    private void setStatus(Status newStatus)
+    {
+        Invariants.checkState((newStatus.permittedFrom & (1 << (status & STATUS_MASK))) != 0, "%s not permitted from %s", newStatus, status());
+        status &= ~STATUS_MASK;
+        status |= newStatus.ordinal();
+        Invariants.checkState(status() == newStatus);
+    }
+
+    public void initialize(V value)
+    {
+        Invariants.checkState(state == null);
+        setStatus(LOADED);
+        state = value;
+    }
+
+    public void readyToLoad()
+    {
+        Invariants.checkState(state == null);
+        setStatus(WAITING_TO_LOAD);
+        state = new WaitingToLoad();
+    }
+
+    public void markNoEvict(int generation, int maxAge)
+    {
+        Invariants.checkState((maxAge & ~0xff) == 0);
+        Invariants.checkState((generation & ~0xffff) == 0);
+        status |= NO_EVICT;
+        status |= generation << 8;
+        status |= maxAge << 24;
+    }
+
+    public LoadingOrWaiting loadingOrWaiting()
+    {
+        return (LoadingOrWaiting)state;
+    }
+
+    void notifyListeners(BiConsumer<AccordCache.Listener<K, V>, AccordCacheEntry<K, V>> notify)
+    {
+        owner.notifyListeners(notify, this);
+    }
+
+    public interface OnLoaded
+    {
+        <K, V> void onLoaded(AccordCacheEntry<K, V> state, V value, Throwable fail);
+
+        static OnLoaded immediate()
+        {
+            return new OnLoaded()
+            {
+                @Override
+                public <K, V> void onLoaded(AccordCacheEntry<K, V> state, V value, Throwable fail)
+                {
+                    if (fail == null) state.loaded(value);
+                    else state.failedToLoad();
+                }
+            };
+        }
+    }
+
+    public interface OnSaved
+    {
+        <K, V> void onSaved(AccordCacheEntry<K, V> state, Object identity, Throwable fail);
+
+        static OnSaved immediate()
+        {
+            return new OnSaved()
+            {
+                @Override
+                public <K, V> void onSaved(AccordCacheEntry<K, V> state, Object identity, Throwable fail)
+                {
+                    state.saved(identity, fail);
+                }
+            };
+        }
+    }
+
+    public <P> Loading load(BiFunction<P, Runnable, Cancellable> loadExecutor, P param, Adapter<K, V, ?> adapter, OnLoaded onLoaded)
+    {
+        Invariants.checkState(is(WAITING_TO_LOAD), "%s", this);
+        Loading loading = ((WaitingToLoad)state).load(loadExecutor.apply(param, () -> {
+            V result;
+            try
+            {
+                result = adapter.load(owner.commandStore, key);
+            }
+            catch (Throwable t)
+            {
+                onLoaded.onLoaded(this, null, t);
+                throw t;
+            }
+            onLoaded.onLoaded(this, result, null);
+        }));
+        setStatus(LOADING);
+        state = loading;
+        return loading;
+    }
+
+    public Loading testLoad()
+    {
+        Invariants.checkState(is(WAITING_TO_LOAD));
+        Loading loading = ((WaitingToLoad)state).load(() -> {});
+        setStatus(LOADING);
+        state = loading;
+        return loading;
+    }
+
+    public Loading loading()
+    {
+        Invariants.checkState(is(LOADING), "%s", this);
+        return (Loading) state;
+    }
+
+    // must own the cache's lock when invoked. this is true of most methods in the class,
+    // but this one is less obvious so named as to draw attention
+    public V getExclusive()
+    {
+        Invariants.checkState(owner == null || owner.commandStore == null || owner.commandStore.executor().isOwningThread());
+        Invariants.checkState(isLoaded(), "%s", this);
+        if (isShrunk())
+        {
+            AccordCache.Type<K, V, ?> parent = owner.parent();
+            inflate(key, parent.adapter());
+            updateSize(parent);
+        }
+
+        return (V)unwrap();
+    }
+
+    private Object unwrap()
+    {
+        return isNested() ? ((Nested)state).state : state;
+    }
+
+    // must own the cache's lock when invoked
+    void setExclusive(V value)
+    {
+        if (value == state)
+            return;
+
+        Saving cancel = is(SAVING) ? ((Saving)state) : null;
+        setStatus(MODIFIED);
+        state = value;
+        updateSize(owner.parent());
+        // TODO (expected): do we want to always cancel in-progress saving?
+        if (cancel != null)
+            cancel.saving.cancel();
+    }
+
+    public void loaded(V value)
+    {
+        setStatus(LOADED);
+        state = value;
+        updateSize(owner.parent());
+    }
+
+    public void testLoaded(V value)
+    {
+        setStatus(LOADED);
+        state = value;
+    }
+
+    public void failedToLoad()
+    {
+        setStatus(FAILED_TO_LOAD);
+        state = null;
+    }
+
+    boolean tryShrink()
+    {
+        if (!isLoaded())
+            return false;
+
+        AccordCache.Type<K, V, ?> parent = owner.parent();
+        if (!tryShrink(key, parent.adapter()))
+            return false;
+        updateSize(parent);
+        return true;
+    }
+
+    V tryGetFull()
+    {
+        return isShrunk() ? null : (V)unwrap();
+    }
+
+    Object tryGetShrunk()
+    {
+        return isShrunk() ? unwrap() : null;
+    }
+
+    boolean isNull()
+    {
+        return state == null;
+    }
+
+    /**
+     * Submits a save runnable to the specified executor. When the runnable
+     * has completed, the state save will have either completed or failed.
+     */
+    @VisibleForTesting
+    void save(Function<Runnable, Cancellable> saveExecutor, Adapter<K, V, ?> adapter, OnSaved onSaved)
+    {
+        V full = isShrunk() ? null : (V)state;
+        Object shrunk = isShrunk() ? state : null;
+        Runnable save = adapter.save(owner.commandStore, key, full, shrunk);
+        if (null == save) // null mutation -> null Runnable -> no change on disk
+        {
+            setStatus(LOADED);
+        }
+        else
+        {
+            setStatus(SAVING);
+            Object identity = new Object();
+            Cancellable saving = saveExecutor.apply(() -> {
+                try
+                {
+                    save.run();
+                }
+                catch (Throwable t)
+                {
+                    onSaved.onSaved(this, identity, t);
+                    throw t;
+                }
+                onSaved.onSaved(this, identity, null);
+            });
+            state = new Saving(saving, identity, state);
+        }
+    }
+
+    boolean saved(Object identity, Throwable fail)
+    {
+        if (!is(SAVING))
+            return false;
+
+        Saving saving = (Saving) state;
+        if (saving.identity != identity)
+            return false;
+
+        if (fail != null)
+        {
+            setStatus(FAILED_TO_SAVE);
+            state = new FailedToSave(fail, ((Saving)state).state);
+            return false;
+        }
+        else
+        {
+            setStatus(LOADED);
+            state = saving.state;
+            return true;
+        }
+    }
+
+    protected void saved()
+    {
+        Invariants.checkState(is(MODIFIED));
+        setStatus(LOADED);
+    }
+
+    public Cancellable saving()
+    {
+        return ((Saving)state).saving;
+    }
+
+    public AccordCacheEntry<K, V> evicted()
+    {
+        setStatus(EVICTED);
+        state = null;
+        return this;
+    }
+
+    public Throwable failure()
+    {
+        return ((FailedToSave)state).cause;
+    }
+
+    private boolean tryShrink(K key, Adapter<K, V, ?> adapter)
+    {
+        Invariants.checkState(!isNested());
+        if (isShrunk() || state == null)
+            return false;
+
+        Object update = adapter.fullShrink(key, (V)state);
+        if (update == null || update == state)
+            return false;
+
+        state = update;
+        status |= SHRUNK;
+        return true;
+    }
+
+    private void inflate(K key, Adapter<K, V, ?> adapter)
+    {
+        Invariants.checkState(isShrunk());
+        if (isNested())
+        {
+            Nested nested = (Nested) state;
+            nested.state = adapter.inflate(key, nested.state);
+        }
+        else
+        {
+            state = adapter.inflate(key, state);
+        }
+        status &= ~SHRUNK;
+    }
+
+    private long estimateOnHeapSize(Adapter<K, V, ?> adapter)
+    {
+        Object current = unwrap();
+        if (current == null) return 0;
+        else if (isShrunk()) return adapter.estimateShrunkHeapSize(current);
+        return adapter.estimateHeapSize((V)current);
+    }
+
+    public static abstract class LoadingOrWaiting
+    {
+        Collection<AccordTask<?>> waiters;
+
+        public LoadingOrWaiting()
+        {
+        }
+
+        public LoadingOrWaiting(Collection<AccordTask<?>> waiters)
+        {
+            this.waiters = waiters;
+        }
+
+        public Collection<AccordTask<?>> waiters()
+        {
+            return waiters != null ? waiters : Collections.emptyList();
+        }
+
+        public BufferList<AccordTask<?>> copyWaiters()
+        {
+            BufferList<AccordTask<?>> list = new BufferList<>();
+            if (waiters != null)
+                list.addAll(waiters);
+            return list;
+        }
+
+        public void add(AccordTask<?> waiter)
+        {
+            if (waiters == null)
+                waiters = new ArrayList<>();
+            waiters.add(waiter);
+        }
+
+        public void remove(AccordTask<?> waiter)
+        {
+            if (waiters != null)
+            {
+                waiters.remove(waiter);
+                if (waiters.isEmpty())
+                    waiters = null;
+            }
+        }
+    }
+
+    static class WaitingToLoad extends LoadingOrWaiting
+    {
+        public Loading load(Cancellable loading)
+        {
+            Invariants.paranoid(waiters == null || !waiters.isEmpty());
+            Loading result = new Loading(waiters, loading);
+            waiters = Collections.emptyList();
+            return result;
+        }
+    }
+
+    static class Loading extends LoadingOrWaiting
+    {
+        public final Cancellable loading;
+
+        public Loading(Collection<AccordTask<?>> waiters, Cancellable loading)
+        {
+            super(waiters);
+            this.loading = loading;
+        }
+    }
+
+    static class Nested
+    {
+        Object state;
+    }
+
+    static class Saving extends Nested
+    {
+        final Cancellable saving;
+        final Object identity;
+
+        Saving(Cancellable saving, Object identity, Object state)
+        {
+            this.saving = saving;
+            this.identity = identity;
+            this.state = state;
+        }
+    }
+
+    static class FailedToSave extends Nested
+    {
+        final Throwable cause;
+
+        FailedToSave(Throwable cause, Object state)
+        {
+            this.cause = cause;
+            this.state = state;
+        }
+
+        public Throwable failure()
+        {
+            return cause;
+        }
+    }
+
+    public static <K, V> AccordCacheEntry<K, V> createReadyToLoad(K key, AccordCache.Type<K, V, ?>.Instance owner)
+    {
+        AccordCacheEntry<K, V> node = new AccordCacheEntry<>(key, owner);
+        node.readyToLoad();
+        return node;
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordCachingState.java b/src/java/org/apache/cassandra/service/accord/AccordCachingState.java
deleted file mode 100644
index 52d6b7c090..0000000000
--- a/src/java/org/apache/cassandra/service/accord/AccordCachingState.java
+++ /dev/null
@@ -1,651 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.util.concurrent.Callable;
-import java.util.function.Function;
-import java.util.function.ToLongFunction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.primitives.Ints;
-
-import accord.utils.IntrusiveLinkedListNode;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncResults.RunnableResult;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.utils.ObjectSizes;
-
-import static java.lang.String.format;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_SAVE;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.MODIFIED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.SAVING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.UNINITIALIZED;
-
-/**
- * Global (per CommandStore) state of a cached entity (Command or CommandsForKey).
- */
-public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
-{
-    static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCachingState<>(null, 0, null));
-
-    public interface Factory<K, V>
-    {
-        AccordCachingState<K, V> create(K key, int index);
-    }
-
-    static <K, V> Factory<K, V> defaultFactory()
-    {
-        return AccordCachingState::new;
-    }
-
-    private final K key;
-    private State<K, V> state;
-
-    int references = 0;
-    int lastQueriedEstimatedSizeOnHeap = 0;
-    final int index;
-    private boolean shouldUpdateSize;
-
-    AccordCachingState(K key, int index)
-    {
-        this.key = key;
-        Invariants.checkArgument(index >= 0);
-        this.index = index;
-        //noinspection unchecked
-        this.state = (State<K, V>) Uninitialized.instance;
-    }
-
-    private AccordCachingState(K key, int index, State<K, V> state)
-    {
-        this.key = key;
-        this.index = index;
-        this.state = state;
-    }
-
-    void unlink()
-    {
-        remove();
-    }
-
-    boolean isLinked()
-    {
-        return !isFree();
-    }
-
-    public K key()
-    {
-        return key;
-    }
-
-    public int referenceCount()
-    {
-        return references;
-    }
-
-    boolean isLoaded()
-    {
-        return status().isLoaded();
-    }
-
-    public boolean isComplete()
-    {
-        return status().isComplete();
-    }
-
-    int lastQueriedEstimatedSizeOnHeap()
-    {
-        return lastQueriedEstimatedSizeOnHeap;
-    }
-
-    int estimatedSizeOnHeap(ToLongFunction<V> estimator)
-    {
-        shouldUpdateSize = false;   // TODO (expected): probably not the safest place to clear need to compute size
-        return lastQueriedEstimatedSizeOnHeap = Ints.checkedCast(EMPTY_SIZE + estimateStateOnHeapSize(estimator));
-    }
-
-    long estimatedSizeOnHeapDelta(ToLongFunction<V> estimator)
-    {
-        long prevSize = lastQueriedEstimatedSizeOnHeap;
-        return estimatedSizeOnHeap(estimator) - prevSize;
-    }
-
-    boolean shouldUpdateSize()
-    {
-        return shouldUpdateSize;
-    }
-
-    @Override
-    public String toString()
-    {
-        return "Node{" + state.status() +
-               ", key=" + key() +
-               ", references=" + references +
-               "}@" + Integer.toHexString(System.identityHashCode(this));
-    }
-
-    public Status status()
-    {
-        return complete().status();
-    }
-
-    State<K, V> complete()
-    {
-        return state.isCompleteable() ? state(state.complete()) : state;
-    }
-
-    /**
-     * Submits a load runnable to the specified executor. When the runnable
-     * has completed, the state load will have either completed or failed.
-     */
-    public AsyncChain<V> load(ExecutorPlus executor, Function<K, V> loadFunction)
-    {
-        Loading<K, V> loading = state.load(key, loadFunction);
-        executor.submit(loading);
-        state(loading);
-        return loading;
-    }
-
-    public void initialize(V value)
-    {
-        state(state.initialize(value));
-    }
-
-    protected State<K, V> state(State<K, V> next)
-    {
-        State<K, V> prev = state;
-        if (prev != next)   // TODO (expected): we change state to transition the cache state machine but often keep payload the same - so shouldn't recompute
-            shouldUpdateSize = true;
-        return state = next;
-    }
-
-    @VisibleForTesting
-    protected State<K, V> state()
-    {
-        return state;
-    }
-
-    public AsyncChain<V> loading()
-    {
-        // do *not* attempt to complete, to prevent races where the caller found a pending load, attempts
-        // to register a callback, but gets an exception because the load completed in the meantime
-        return state.loading();
-    }
-
-    public V get()
-    {
-        return complete().get();
-    }
-
-    public void set(V value)
-    {
-        shouldUpdateSize = true;
-        state(complete().set(value));
-    }
-
-    /**
-     * Submits a save runnable to the specified executor. When the runnable
-     * has completed, the state save will have either completed or failed.
-     */
-    @VisibleForTesting
-    public void save(ExecutorPlus executor, Function<?, Runnable> saveFunction)
-    {
-        @SuppressWarnings("unchecked")
-        State<K, V> savingOrLoaded = state.save((Function<V, Runnable>) saveFunction);
-        if (savingOrLoaded.status() == SAVING)
-            executor.submit(savingOrLoaded.saving());
-        state(savingOrLoaded);
-    }
-
-    public AsyncChain<Void> saving()
-    {
-        // do *not* attempt to complete, to prevent races where the caller found a pending save, attempts
-        // to register a callback, but gets an exception because the save completed in the meantime
-        return state.saving();
-    }
-
-    public AccordCachingState<K, V> reset()
-    {
-        state(state.reset());
-        return this;
-    }
-
-    public Throwable failure()
-    {
-        return complete().failure();
-    }
-
-    public void markEvicted()
-    {
-        state(complete().evict());
-        lastQueriedEstimatedSizeOnHeap = 0;
-        shouldUpdateSize = false;
-    }
-
-    long estimateStateOnHeapSize(ToLongFunction<V> estimateFunction)
-    {
-        return state.estimateOnHeapSize(estimateFunction);
-    }
-
-    public enum Status
-    {
-        UNINITIALIZED,
-        LOADING,
-        LOADED,
-        FAILED_TO_LOAD,
-        MODIFIED,
-        SAVING,
-
-        /**
-         * Attempted to save but failed. Shouldn't normally happen unless we have a bug in serialization,
-         * or commit log has been stopped.
-         */
-        FAILED_TO_SAVE,
-
-        /**
-         * Entry has been successfully evicted, but there were transient listeners present, so we kept the
-         * Node around (transient listeners must survive cache eviction).
-         */
-        EVICTED,
-        ;
-
-        boolean isLoaded()
-        {
-            return this == LOADED || this == MODIFIED || this == FAILED_TO_SAVE;
-        }
-
-        boolean isComplete()
-        {
-            return !(this == LOADING || this == SAVING);
-        }
-    }
-
-    interface State<K, V>
-    {
-        Status status();
-
-        default boolean isCompleteable()
-        {
-            return false;
-        }
-
-        default State<K, V> complete()
-        {
-            throw illegalState(this, "complete()");
-        }
-
-        default Loading<K, V> load(K key, Function<K, V> loadFunction)
-        {
-            throw illegalState(this, "load(key, loadFunction)");
-        }
-
-        default Loaded<K, V> initialize(V value)
-        {
-            throw illegalState(this, "initialize(value)");
-        }
-
-        default RunnableResult<V> loading()
-        {
-            throw illegalState(this, "loading()");
-        }
-
-        default V get()
-        {
-            throw illegalState(this, "get()");
-        }
-
-        default State<K, V> set(V value)
-        {
-            throw illegalState(this, "set(value)");
-        }
-
-        default State<K, V> save(Function<V, Runnable> saveFunction)
-        {
-            throw illegalState(this, "save(saveFunction)");
-        }
-
-        default RunnableResult<Void> saving()
-        {
-            throw illegalState(this, "saving()");
-        }
-
-        default Throwable failure()
-        {
-            throw illegalState(this, "failure()");
-        }
-
-        default Uninitialized<K, V> reset()
-        {
-            throw illegalState(this, "reset()");
-        }
-
-        default Evicted<K, V> evict()
-        {
-            throw illegalState(this, "evict()");
-        }
-
-        default long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return 0;
-        }
-    }
-
-    private static IllegalStateException illegalState(State<?, ?> state, String method)
-    {
-        return new IllegalStateException(format("%s invoked on %s", method, state.status()));
-    }
-
-    static class Uninitialized<K, V> implements State<K, V>
-    {
-        static final Uninitialized<?, ?> instance = new Uninitialized<>();
-
-        @SuppressWarnings("unchecked")
-        static <K, V> Uninitialized<K, V> instance()
-        {
-            return (Uninitialized<K, V>) instance;
-        }
-
-        @Override
-        public Status status()
-        {
-            return UNINITIALIZED;
-        }
-
-        @Override
-        public Loading<K, V> load(K key, Function<K, V> loadFunction)
-        {
-            return new Loading<>(() -> loadFunction.apply(key));
-        }
-
-        public Loaded<K, V> initialize(V value)
-        {
-            return new Loaded<>(value);
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-    }
-
-    static class Loading<K, V> extends RunnableResult<V> implements State<K, V>
-    {
-        Loading(Callable<V> callable)
-        {
-            super(callable);
-        }
-
-        @Override
-        public Status status()
-        {
-            return LOADING;
-        }
-
-        @Override
-        public boolean isCompleteable()
-        {
-            return isDone();
-        }
-
-        @Override
-        public State<K, V> complete()
-        {
-            if      (!isDone())   return this;
-            else if (isSuccess()) return new Loaded<>(result());
-            else                  return new FailedToLoad<>(failure());
-        }
-
-        @Override
-        public RunnableResult<V> loading()
-        {
-            return this;
-        }
-    }
-
-    static class Loaded<K, V> implements State<K, V>
-    {
-        final V original;
-
-        Loaded(V original)
-        {
-            this.original = original;
-        }
-
-        @Override
-        public Status status()
-        {
-            return LOADED;
-        }
-
-        @Override
-        public V get()
-        {
-            return original;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            return value == original ? this : new Modified<>(value);
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-
-        @Override
-        public long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return null == original ? 0 : estimateFunction.applyAsLong(original);
-        }
-    }
-
-    static class FailedToLoad<K, V> implements State<K, V>
-    {
-        final Throwable cause;
-
-        FailedToLoad(Throwable cause)
-        {
-            this.cause = cause;
-        }
-
-        @Override
-        public Status status()
-        {
-            return FAILED_TO_LOAD;
-        }
-
-        @Override
-        public Throwable failure()
-        {
-            return cause;
-        }
-
-        @Override
-        public Uninitialized<K, V> reset()
-        {
-            return Uninitialized.instance();
-        }
-
-        @Override
-        public Evicted<K, V> evict()
-        {
-            return Evicted.instance();
-        }
-    }
-
-    static class Modified<K, V> implements State<K, V>
-    {
-        V current;
-
-        Modified(V current)
-        {
-            this.current = current;
-        }
-
-        @Override
-        public Status status()
-        {
-            return MODIFIED;
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            current = value;
-            return this;
-        }
-
-        @Override
-        public State<K, V> save(Function<V, Runnable> saveFunction)
-        {
-            Runnable runnable = saveFunction.apply(current);
-            if (null == runnable) // null mutation -> null Runnable -> no change on disk
-                return new Loaded<>(current);
-            else
-                return new Saving<>(current, runnable);
-        }
-
-        @Override
-        public long estimateOnHeapSize(ToLongFunction<V> estimateFunction)
-        {
-            return (null == current  ? 0 : estimateFunction.applyAsLong(current));
-        }
-    }
-
-    static class Saving<K, V> extends RunnableResult<Void> implements State<K, V>
-    {
-        V current;
-
-        Saving(V current, Runnable saveRunnable)
-        {
-            this(current, () -> { saveRunnable.run(); return null; });
-        }
-
-        Saving(V current, Callable<Void> saveCallable)
-        {
-            super(saveCallable);
-            this.current = current;
-        }
-
-        @Override
-        public Status status()
-        {
-            return SAVING;
-        }
-
-        @Override
-        public boolean isCompleteable()
-        {
-            return isDone();
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> complete()
-        {
-            if      (!isDone())   return this;
-            else if (isSuccess()) return new Loaded<>(current);
-            else                  return new FailedToSave<>(current, failure());
-        }
-
-        @Override
-        public RunnableResult<Void> saving()
-        {
-            return this;
-        }
-    }
-
-    static class FailedToSave<K, V> implements State<K, V>
-    {
-        V current;
-        final Throwable cause;
-
-        FailedToSave(V current, Throwable cause)
-        {
-            this.current = current;
-            this.cause = cause;
-        }
-
-        @Override
-        public Status status()
-        {
-            return FAILED_TO_SAVE;
-        }
-
-        @Override
-        public V get()
-        {
-            return current;
-        }
-
-        @Override
-        public State<K, V> set(V value)
-        {
-            current = value;
-            return this;
-        }
-
-        @Override
-        public Throwable failure()
-        {
-            return cause;
-        }
-    }
-
-    static class Evicted<K, V> implements State<K, V>
-    {
-        static final Evicted<?, ?> instance = new Evicted<>();
-
-        @SuppressWarnings("unchecked")
-        static <K, V> Evicted<K, V> instance()
-        {
-            return (Evicted<K, V>) instance;
-        }
-
-        @Override
-        public Status status()
-        {
-            return EVICTED;
-        }
-
-        @Override
-        public Uninitialized<K, V> reset()
-        {
-            return Uninitialized.instance();
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
index c7ee8aa6cd..4394fbb29a 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
@@ -18,24 +18,20 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.IdentityHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Objects;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntFunction;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import accord.api.Agent;
 import accord.api.DataStore;
@@ -55,6 +51,7 @@ import accord.local.RedundantBefore;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.cfk.CommandsForKey;
+import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
 import accord.primitives.RangeDeps;
 import accord.primitives.Ranges;
@@ -66,22 +63,17 @@ import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
-import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.SequentialExecutorPlus;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.service.accord.SavedCommand.MinimalCommand;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.service.accord.async.AsyncOperation;
-import org.apache.cassandra.service.accord.events.CacheEvents;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Promise;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
-import static accord.local.KeyHistory.COMMANDS;
 import static accord.primitives.SaveStatus.Applying;
+import static accord.local.KeyHistory.SYNC;
 import static accord.primitives.Status.Committed;
 import static accord.primitives.Status.Invalidated;
 import static accord.primitives.Status.Truncated;
@@ -90,87 +82,70 @@ import static org.apache.cassandra.service.accord.SavedCommand.Load.MINIMAL;
 
 public class AccordCommandStore extends CommandStore
 {
-    private static final Logger logger = LoggerFactory.getLogger(AccordCommandStore.class);
-    private static final boolean CHECK_THREADS = CassandraRelevantProperties.TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED.getBoolean();
+    // TODO (required): track this via a PhantomReference, so that if we remove a CommandStore without clearing the caches we can be sure to release them
+    public static class Caches
+    {
+        private final AccordCache global;
+        private final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands;
+        private final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys;
+        private final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys;
+
+        Caches(AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commandCache, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeyCache, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeyCache)
+        {
+            this.global = global;
+            this.commands = commandCache;
+            this.timestampsForKeys = timestampsForKeyCache;
+            this.commandsForKeys = commandsForKeyCache;
+        }
+
+        public final AccordCache global()
+        {
+            return global;
+        }
+
+        public final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands()
+        {
+            return commands;
+        }
+
+        public final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys()
+        {
+            return timestampsForKeys;
+        }
+
+        public final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys()
+        {
+            return commandsForKeys;
+        }
+    }
+
+    public static final class ExclusiveCaches extends Caches implements AutoCloseable
+    {
+        private final Lock lock;
+
+        public ExclusiveCaches(Lock lock, AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKeys, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeys)
+        {
+            super(global, commands, timestampsForKeys, commandsForKeys);
+            this.lock = lock;
+        }
+
+        @Override
+        public void close()
+        {
+            lock.unlock();
+        }
+    }
 
     public final String loggingId;
     private final IJournal journal;
-
-    private final CommandStoreExecutor executor;
-    private final AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache;
-    private final AccordStateCache.Instance<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache;
-    private final AccordStateCache.Instance<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache;
-    private AsyncOperation<?> currentOperation = null;
-    private AccordSafeCommandStore current = null;
+    private final AccordExecutor executor;
+    private final Executor taskExecutor;
+    private final ExclusiveCaches caches;
     private long lastSystemTimestampMicros = Long.MIN_VALUE;
     private final CommandsForRangesLoader commandsForRangesLoader;
 
-    private static <K, V> void registerJfrListener(int id, AccordStateCache.Instance<K, V, ?> instance, String name)
-    {
-        if (!DatabaseDescriptor.getAccordStateCacheListenerJFREnabled())
-            return;
-        instance.register(new AccordStateCache.Listener<>() {
-            private final IdentityHashMap<AccordCachingState<?, ?>, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>();
-
-            @Override
-            public void onAdd(AccordCachingState<K, V> state)
-            {
-                CacheEvents.Add add = new CacheEvents.Add();
-                CacheEvents.Evict evict = new CacheEvents.Evict();
-                if (!add.isEnabled())
-                    return;
-                add.begin();
-                evict.begin();
-                add.store = evict.store = id;
-                add.instance = evict.instance = name;
-                add.key = evict.key = state.key().toString();
-                updateMutable(instance, state, add);
-                add.commit();
-                pendingEvicts.put(state, evict);
-            }
-
-            @Override
-            public void onRelease(AccordCachingState<K, V> state)
-            {
-
-            }
-
-            @Override
-            public void onEvict(AccordCachingState<K, V> state)
-            {
-                CacheEvents.Evict event = pendingEvicts.remove(state);
-                if (event == null) return;
-                updateMutable(instance, state, event);
-                event.commit();
-            }
-        });
-    }
-
-    private static void updateMutable(AccordStateCache.Instance<?, ?, ?> instance, AccordCachingState<?, ?> state, CacheEvents event)
-    {
-        event.status = state.state().status().name();
-
-        event.lastQueriedEstimatedSizeOnHeap = state.lastQueriedEstimatedSizeOnHeap();
-
-        event.instanceAllocated = instance.weightedSize();
-        AccordStateCache.Stats stats = instance.stats();
-        event.instanceStatsQueries = stats.queries;
-        event.instanceStatsHits = stats.hits;
-        event.instanceStatsMisses = stats.misses;
-
-        event.globalSize = instance.size();
-        event.globalReferenced = instance.globalReferencedEntries();
-        event.globalUnreferenced = instance.globalUnreferencedEntries();
-        event.globalCapacity = instance.capacity();
-        event.globalAllocated = instance.globalAllocated();
-
-        stats = instance.globalStats();
-        event.globalStatsQueries = stats.queries;
-        event.globalStatsHits = stats.hits;
-        event.globalStatsMisses = stats.misses;
-
-        event.update();
-    }
+    private AccordSafeCommandStore current;
+    private Thread currentThread;
 
     public AccordCommandStore(int id,
                               NodeCommandStoreService node,
@@ -180,53 +155,33 @@ public class AccordCommandStore extends CommandStore
                               LocalListeners.Factory listenerFactory,
                               EpochUpdateHolder epochUpdateHolder,
                               IJournal journal,
-                              CommandStoreExecutor commandStoreExecutor)
+                              AccordExecutor executor)
     {
         super(id, node, agent, dataStore, progressLogFactory, listenerFactory, epochUpdateHolder);
-        this.journal = journal;
         loggingId = String.format("[%s]", id);
-        executor = commandStoreExecutor;
-        AccordStateCache stateCache = executor.stateCache;
-        commandCache =
-            stateCache.instance(TxnId.class,
-                                AccordSafeCommand.class,
-                                AccordSafeCommand.safeRefFactory(),
-                                this::loadCommand,
-                                this::appendToKeyspace,
-                                this::validateCommand,
-                                AccordObjectSizes::command);
-        registerJfrListener(id, commandCache, "Command");
-        timestampsForKeyCache =
-            stateCache.instance(RoutingKey.class,
-                                AccordSafeTimestampsForKey.class,
-                                AccordSafeTimestampsForKey::new,
-                                this::loadTimestampsForKey,
-                                this::saveTimestampsForKey,
-                                this::validateTimestampsForKey,
-                                AccordObjectSizes::timestampsForKey);
-        registerJfrListener(id, timestampsForKeyCache, "TimestampsForKey");
-        commandsForKeyCache =
-            stateCache.instance(RoutingKey.class,
-                                AccordSafeCommandsForKey.class,
-                                AccordSafeCommandsForKey::new,
-                                this::loadCommandsForKey,
-                                this::saveCommandsForKey,
-                                this::validateCommandsForKey,
-                                AccordObjectSizes::commandsForKey,
-                                AccordCachingState::new);
-        registerJfrListener(id, commandsForKeyCache, "CommandsForKey");
+        this.journal = journal;
+        this.executor = executor;
 
+        final AccordCache.Type<TxnId, Command, AccordSafeCommand>.Instance commands;
+        final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey>.Instance timestampsForKey;
+        final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKey;
+        try (AccordExecutor.ExclusiveGlobalCaches exclusive = executor.lockCaches())
+        {
+            commands = exclusive.commands.newInstance(this);
+            timestampsForKey = exclusive.timestampsForKey.newInstance(this);
+            commandsForKey = exclusive.commandsForKey.newInstance(this);
+            this.caches = new ExclusiveCaches(executor.lock, exclusive.global, commands, timestampsForKey, commandsForKey);
+        }
+
+        this.taskExecutor = executor.executor(this);
         this.commandsForRangesLoader = new CommandsForRangesLoader(this);
-
         loadRedundantBefore(journal.loadRedundantBefore(id()));
         loadBootstrapBeganAt(journal.loadBootstrapBeganAt(id()));
         loadSafeToRead(journal.loadSafeToRead(id()));
         loadRangesForEpoch(journal.loadRangesForEpoch(id()));
-
-        executor.execute(() -> CommandStore.register(this));
     }
 
-    static Factory factory(AccordJournal journal, IntFunction<CommandStoreExecutor> executorFactory)
+    static Factory factory(AccordJournal journal, IntFunction<AccordExecutor> executorFactory)
     {
         return (id, node, agent, dataStore, progressLogFactory, listenerFactory, rangesForEpoch) ->
                new AccordCommandStore(id, node, agent, dataStore, progressLogFactory, listenerFactory, rangesForEpoch, journal, executorFactory.apply(id));
@@ -247,61 +202,65 @@ public class AccordCommandStore extends CommandStore
     @Override
     public boolean inStore()
     {
-        return executor.isInThread();
+        return currentThread == Thread.currentThread();
     }
 
-    public void checkInStoreThread()
+    void tryPreSetup(AccordTask<?> task)
     {
-        checkState(inStore());
+        if (inStore() && current != null)
+            task.presetup(current.task);
     }
 
-    public void checkNotInStoreThread()
+    public AccordExecutor executor()
     {
-        if (!CHECK_THREADS)
-            return;
-        checkState(!inStore());
+        return executor;
     }
 
-    public ExecutorService executor()
+    // TODO (desired): we use this for executing callbacks with mutual exclusivity,
+    //  but we don't need to block the actual CommandStore - could quite easily
+    //  inflate a separate queue dynamically in AccordExecutor
+    public Executor taskExecutor()
     {
-        return executor.delegate();
+        return taskExecutor;
     }
 
-    /**
-     * Note that this cache is shared with other commandStores!
-     */
-    public AccordStateCache cache()
+    public ExclusiveCaches lockCaches()
     {
-        return executor.cache();
+        //noinspection LockAcquiredButNotSafelyReleased
+        caches.lock.lock();
+        return caches;
     }
 
-    public AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache()
+    public ExclusiveCaches tryLockCaches()
     {
-        return commandCache;
+        if (caches.lock.tryLock())
+            return caches;
+        return null;
     }
 
-    public AccordStateCache.Instance<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache()
+    public Caches cachesExclusive()
     {
-        return timestampsForKeyCache;
+        Invariants.checkState(executor.isOwningThread());
+        return caches;
     }
 
-    public AccordStateCache.Instance<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache()
+    public Caches cachesUnsafe()
     {
-        return commandsForKeyCache;
+        return caches;
     }
 
     @VisibleForTesting
     @Override
-    protected void unsafeSetRangesForEpoch(CommandStores.RangesForEpoch newRangesForEpoch)
+    public void unsafeSetRangesForEpoch(CommandStores.RangesForEpoch newRangesForEpoch)
     {
         super.unsafeSetRangesForEpoch(newRangesForEpoch);
     }
 
     @Nullable
     @VisibleForTesting
-    public Runnable appendToKeyspace(Command after)
+    public Runnable appendToKeyspace(TxnId txnId, Command after)
     {
-        if (after.txnId().is(Routable.Domain.Key))
+        if (txnId.is(Routable.Domain.Key))
             return null;
 
         Mutation mutation = AccordKeyspace.getCommandMutation(this.id, after, nextSystemTimestampMicros());
@@ -345,18 +304,18 @@ public class AccordCommandStore extends CommandStore
         if (!Invariants.isParanoid())
             return true;
 
-        TimestampsForKey reloaded = AccordKeyspace.unsafeLoadTimestampsForKey(this, (TokenKey) key);
+        TimestampsForKey reloaded = AccordKeyspace.unsafeLoadTimestampsForKey(id, (TokenKey) key);
         return Objects.equals(evicting, reloaded);
     }
 
     TimestampsForKey loadTimestampsForKey(RoutableKey key)
     {
-        return AccordKeyspace.loadTimestampsForKey(this, (TokenKey) key);
+        return AccordKeyspace.loadTimestampsForKey(id, (TokenKey) key);
     }
 
     CommandsForKey loadCommandsForKey(RoutableKey key)
     {
-        return AccordKeyspace.loadCommandsForKey(this, (TokenKey) key);
+        return AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
     }
 
     boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting)
@@ -364,40 +323,20 @@ public class AccordCommandStore extends CommandStore
         if (!Invariants.isParanoid())
             return true;
 
-        CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(this, (TokenKey) key);
+        CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
         return Objects.equals(evicting, reloaded);
     }
 
     @Nullable
-    private Runnable saveTimestampsForKey(TimestampsForKey after)
+    Runnable saveTimestampsForKey(RoutingKey key, TimestampsForKey after, Object serialized)
     {
-        Mutation mutation = AccordKeyspace.getTimestampsForKeyMutation(id, after, nextSystemTimestampMicros());
-        return null != mutation ? mutation::applyUnsafe : null;
+        return AccordKeyspace.getTimestampsForKeyUpdater(this, after, nextSystemTimestampMicros());
     }
 
     @Nullable
-    private Runnable saveCommandsForKey(CommandsForKey after)
+    Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized)
     {
-        Mutation mutation = AccordKeyspace.getCommandsForKeyMutation(id, after, nextSystemTimestampMicros());
-        return null != mutation ? mutation::applyUnsafe : null;
-    }
-
-    public void setCurrentOperation(AsyncOperation<?> operation)
-    {
-        checkState(currentOperation == null);
-        currentOperation = operation;
-    }
-
-    public AsyncOperation<?> getContext()
-    {
-        checkState(currentOperation != null);
-        return currentOperation;
-    }
-
-    public void unsetCurrentOperation(AsyncOperation<?> operation)
-    {
-        checkState(currentOperation == operation);
-        currentOperation = null;
+        return AccordKeyspace.getCommandsForKeyUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros());
     }
 
     public long nextSystemTimestampMicros()
@@ -408,13 +347,13 @@ public class AccordCommandStore extends CommandStore
     @Override
     public <T> AsyncChain<T> submit(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
     {
-        return AsyncOperation.create(this, loadCtx, function);
+        return AccordTask.create(this, loadCtx, function).chain();
     }
 
     @Override
     public <T> AsyncChain<T> submit(Callable<T> task)
     {
-        return AsyncChains.ofCallable(executor.delegate(), task);
+        return AsyncChains.ofCallable(taskExecutor(), task);
     }
 
     public DataStore dataStore()
@@ -435,7 +374,7 @@ public class AccordCommandStore extends CommandStore
     @Override
     public AsyncChain<Void> execute(PreLoadContext preLoadContext, Consumer<? super SafeCommandStore> consumer)
     {
-        return AsyncOperation.create(this, preLoadContext, consumer);
+        return AccordTask.create(this, preLoadContext, consumer).chain();
     }
 
     public void executeBlocking(Runnable runnable)
@@ -454,43 +393,38 @@ public class AccordCommandStore extends CommandStore
         }
     }
 
-    public AccordSafeCommandStore beginOperation(PreLoadContext preLoadContext,
-                                                 Map<TxnId, AccordSafeCommand> commands,
-                                                 Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKeys,
-                                                 Map<RoutingKey, AccordSafeCommandsForKey> commandsForKeys,
-                                                 @Nullable AccordSafeCommandsForRanges commandsForRanges)
+    public AccordSafeCommandStore begin(AccordTask<?> operation,
+                                        @Nullable CommandsForRanges commandsForRanges)
     {
         checkState(current == null);
-        commands.values().forEach(AccordSafeState::preExecute);
-        commandsForKeys.values().forEach(AccordSafeState::preExecute);
-        timestampsForKeys.values().forEach(AccordSafeState::preExecute);
-        if (commandsForRanges != null)
-            commandsForRanges.preExecute();
-
-        current = AccordSafeCommandStore.create(preLoadContext, commands, timestampsForKeys, commandsForKeys, commandsForRanges, this);
+        current = AccordSafeCommandStore.create(operation, commandsForRanges, this);
         return current;
     }
 
+    void setOwner(Thread thread, Thread self)
+    {
+        Invariants.checkState(thread == null ? currentThread == self : currentThread == null);
+        currentThread = thread;
+        if (thread != null) CommandStore.register(this);
+
+    }
+
     public boolean hasSafeStore()
     {
         return current != null;
     }
 
-    public void completeOperation(AccordSafeCommandStore store)
+    public void complete(AccordSafeCommandStore store)
     {
         checkState(current == store);
-        try
-        {
-            current.postExecute();
-        }
-        finally
-        {
-            current = null;
-        }
+        current.postExecute();
+        current = null;
     }
 
-    public void abortCurrentOperation()
+    public void abort(AccordSafeCommandStore store)
     {
+        checkInStore();
+        Invariants.checkState(store == current);
         current = null;
     }
 
@@ -511,34 +445,37 @@ public class AccordCommandStore extends CommandStore
         Ranges allRanges = safeStore.ranges().all();
         Ranges coordinateRanges = Ranges.EMPTY;
         long coordinateEpoch = -1;
-        for (int i = 0; i < rangeDeps.txnIdCount(); i++)
+        try (ExclusiveCaches caches = lockCaches())
         {
-            TxnId txnId = rangeDeps.txnId(i);
-            AccordCachingState<TxnId, Command> state = commandCache.getUnsafe(txnId);
-            if (state != null && state.isLoaded() && state.get() != null && state.get().known().isDefinitionKnown())
-                continue;
-
-            Ranges addRanges = rangeDeps.ranges(i).slice(allRanges);
-            if (addRanges.isEmpty()) continue;
-
-            if (coordinateEpoch != txnId.epoch())
+            for (int i = 0; i < rangeDeps.txnIdCount(); i++)
             {
-                coordinateEpoch = txnId.epoch();
-                coordinateRanges = ranges.allAt(txnId.epoch());
+                TxnId txnId = rangeDeps.txnId(i);
+                AccordCacheEntry<TxnId, Command> state = caches.commands().getUnsafe(txnId);
+                if (state != null && state.isLoaded() && state.getExclusive() != null && state.getExclusive().known().isDefinitionKnown())
+                    continue;
+
+                Ranges addRanges = rangeDeps.ranges(i).slice(allRanges);
+                if (addRanges.isEmpty()) continue;
+
+                if (coordinateEpoch != txnId.epoch())
+                {
+                    coordinateEpoch = txnId.epoch();
+                    coordinateRanges = ranges.allAt(txnId.epoch());
+                }
+                if (addRanges.intersects(coordinateRanges)) continue;
+                addRanges = redundantBefore.removeShardRedundant(txnId, txnId, addRanges);
+                if (addRanges.isEmpty()) continue;
+                diskCommandsForRanges().mergeTransitive(txnId, addRanges, Ranges::with);
             }
-            if (addRanges.intersects(coordinateRanges)) continue;
-            addRanges = redundantBefore.removeShardRedundant(txnId, txnId, addRanges);
-            if (addRanges.isEmpty()) continue;
-            diskCommandsForRanges().mergeTransitive(txnId, addRanges, Ranges::with);
         }
     }
 
-    public void appendCommands(List<SavedCommand.DiffWriter> diffs, Runnable onFlush)
+    public void appendCommands(List<SavedCommand.Writer> diffs, Runnable onFlush)
     {
         for (int i = 0; i < diffs.size(); i++)
         {
             boolean isLast = i == diffs.size() - 1;
-            SavedCommand.DiffWriter writer = diffs.get(i);
+            SavedCommand.Writer writer = diffs.get(i);
             journal.appendCommand(id, writer, isLast  ? onFlush : null);
         }
     }
@@ -549,6 +486,21 @@ public class AccordCommandStore extends CommandStore
         return journal.loadCommand(id, txnId, unsafeGetRedundantBefore(), durableBefore());
     }
 
+    public static Command prepareToCache(Command command)
+    {
+        // TODO (required): validate we don't have duplicate objects
+        if (command != null)
+        {
+            PartialTxn txn = command.partialTxn();
+            if (txn != null)
+            {
+                TxnRead read = (TxnRead) txn.read();
+                read.unmemoize();
+            }
+        }
+        return command;
+    }
+
     public MinimalCommand loadMinimal(TxnId txnId)
     {
         return journal.loadMinimal(id, txnId, MINIMAL, unsafeGetRedundantBefore(), durableBefore());
@@ -568,22 +520,13 @@ public class AccordCommandStore extends CommandStore
             {
                 TxnId txnId = command.txnId();
                 Participants<?> keys = null;
-                List<TxnId> deps = null;
                 if (CommandsForKey.manages(txnId))
                     keys = command.hasBeen(Committed) ? command.participants().hasTouched() : command.participants().touches();
                 else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
                     keys = command.asCommitted().waitingOn.keys;
 
-                if (command.partialDeps() != null)
-                    deps = command.partialDeps().txnIds();
-
                 if (keys != null)
-                {
-                    if (deps != null)
-                        return PreLoadContext.contextFor(txnId, deps, keys, keyHistory);
-
                     return PreLoadContext.contextFor(txnId, keys, keyHistory);
-                }
 
                 return PreLoadContext.contextFor(txnId);
             }
@@ -593,12 +536,12 @@ public class AccordCommandStore extends CommandStore
                 TxnId txnId = command.txnId();
 
                 AsyncPromise<?> future = new AsyncPromise<>();
-                execute(context(command, COMMANDS),
+                execute(context(command, SYNC),
                         safeStore -> {
                             Command local = command;
                             if (local.status() != Truncated && local.status() != Invalidated)
                             {
-                                Cleanup cleanup = Cleanup.shouldCleanup(local, unsafeGetRedundantBefore(), durableBefore());
+                                Cleanup cleanup = Cleanup.shouldCleanup(agent, local, unsafeGetRedundantBefore(), durableBefore());
                                 switch (cleanup)
                                 {
                                     case NO:
@@ -678,116 +621,4 @@ public class AccordCommandStore extends CommandStore
         if (rangesForEpoch != null)
             unsafeSetRangesForEpoch(new CommandStores.RangesForEpoch(rangesForEpoch.epochs, rangesForEpoch.ranges, this));
     }
-
-    public static class CommandStoreExecutor implements CacheSize
-    {
-        final AccordStateCache stateCache;
-        final SequentialExecutorPlus delegate;
-        final long threadId;
-
-        CommandStoreExecutor(AccordStateCache stateCache, SequentialExecutorPlus delegate, long threadId)
-        {
-            this.stateCache = stateCache;
-            this.delegate = delegate;
-            this.threadId = threadId;
-        }
-
-        public boolean hasTasks()
-        {
-            return delegate.getPendingTaskCount() > 0 || delegate.getActiveTaskCount() > 0;
-        }
-
-        CommandStoreExecutor(AccordStateCache stateCache, SequentialExecutorPlus delegate)
-        {
-            this.stateCache = stateCache;
-            this.delegate = delegate;
-            this.threadId = getThreadId();
-        }
-
-        public boolean isInThread()
-        {
-            if (!CHECK_THREADS)
-                return true;
-
-            return threadId == Thread.currentThread().getId();
-        }
-
-        public void shutdown()
-        {
-            delegate.shutdown();
-        }
-
-        public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
-        {
-            return delegate.awaitTermination(timeout, unit);
-        }
-
-        public Future<?> submit(Runnable task)
-        {
-            return delegate.submit(task);
-        }
-
-        public ExecutorService delegate()
-        {
-            return delegate;
-        }
-
-        public void execute(Runnable command)
-        {
-            delegate.submit(command);
-        }
-
-        private long getThreadId()
-        {
-            try
-            {
-                return delegate.submit(() -> Thread.currentThread().getId()).get();
-            }
-            catch (InterruptedException e)
-            {
-                throw new AssertionError(e);
-            }
-            catch (ExecutionException e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @VisibleForTesting
-        public AccordStateCache cache()
-        {
-            return stateCache;
-        }
-
-        @VisibleForTesting
-        public void unsafeClearCache()
-        {
-            stateCache.unsafeClear();
-        }
-
-        @Override
-        public void setCapacity(long bytes)
-        {
-            Invariants.checkState(isInThread());
-            stateCache.setCapacity(bytes);
-        }
-
-        @Override
-        public long capacity()
-        {
-            return stateCache.capacity();
-        }
-
-        @Override
-        public int size()
-        {
-            return stateCache.size();
-        }
-
-        @Override
-        public long weightedSize()
-        {
-            return stateCache.weightedSize();
-        }
-    }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java
index d51223243d..0a530bf4d3 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -27,7 +28,6 @@ import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.LocalListeners;
 import accord.api.ProgressLog;
-import accord.local.CommandStore;
 import accord.local.CommandStores;
 import accord.local.Node;
 import accord.local.NodeCommandStoreService;
@@ -36,50 +36,87 @@ import accord.primitives.Range;
 import accord.topology.Topology;
 import accord.utils.RandomSource;
 import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.AccordSpec.QueueShardModel;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
 import org.apache.cassandra.metrics.CacheSizeMetrics;
 import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.accord.AccordCommandStore.CommandStoreExecutor;
+import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.concurrent.Stage.ACCORD_MIGRATION;
-import static org.apache.cassandra.concurrent.Stage.ACCORD_RANGE_LOADER;
-import static org.apache.cassandra.concurrent.Stage.MUTATION;
-import static org.apache.cassandra.concurrent.Stage.READ;
+import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_PER_SHARD;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordQueueSubmissionModel;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordQueueShardCount;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+import static org.apache.cassandra.service.accord.AccordExecutor.constant;
+import static org.apache.cassandra.service.accord.AccordExecutor.constantFactory;
 
 public class AccordCommandStores extends CommandStores implements CacheSize
 {
     public static final String ACCORD_STATE_CACHE = "AccordStateCache";
 
     private final CacheSizeMetrics cacheSizeMetrics;
-    private final CommandStoreExecutor[] executors;
-    private long cacheSize;
+    private final AccordExecutor[] executors;
+    private long cacheSize, workingSetSize;
+    private int maxQueuedLoads, maxQueuedRangeLoads;
+    private boolean shrinkingOn;
 
     AccordCommandStores(NodeCommandStoreService node, Agent agent, DataStore store, RandomSource random,
                         ShardDistributor shardDistributor, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenerFactory,
-                        AccordJournal journal, CommandStoreExecutor[] executors)
+                        AccordJournal journal, AccordExecutor[] executors)
     {
         super(node, agent, store, random, shardDistributor, progressLogFactory, listenerFactory,
               AccordCommandStore.factory(journal, id -> executors[id % executors.length]));
-        setCapacity(DatabaseDescriptor.getAccordCacheSizeInMiB() << 20);
         this.executors = executors;
         this.cacheSizeMetrics = new CacheSizeMetrics(ACCORD_STATE_CACHE, this);
+        cacheSize = DatabaseDescriptor.getAccordCacheSizeInMiB() << 20;
+        workingSetSize = DatabaseDescriptor.getAccordWorkingSetSizeInMiB() << 20;
+        maxQueuedLoads = DatabaseDescriptor.getAccordMaxQueuedLoadCount();
+        maxQueuedRangeLoads = DatabaseDescriptor.getAccordMaxQueuedRangeLoadCount();
+        shrinkingOn = DatabaseDescriptor.getAccordCacheShrinkingOn();
+        refreshCapacities();
     }
 
     static Factory factory(AccordJournal journal)
     {
         return (time, agent, store, random, shardDistributor, progressLogFactory, listenerFactory) -> {
-            CommandStoreExecutor[] executors = new CommandStoreExecutor[DatabaseDescriptor.getAccordShardCount()];
+            AccordExecutor[] executors = new AccordExecutor[getAccordQueueShardCount()];
+            AccordExecutorFactory factory;
+            int maxThreads = Integer.MAX_VALUE;
+            switch (getAccordQueueSubmissionModel())
+            {
+                default: throw new AssertionError("Unhandled QueueSubmissionModel: " + getAccordQueueSubmissionModel());
+                case SYNC: factory = AccordExecutorSyncSubmit::new; break;
+                case SEMI_SYNC: factory = AccordExecutorSemiSyncSubmit::new; break;
+                case ASYNC: factory = AccordExecutorAsyncSubmit::new; break;
+                case EXEC_ST:
+                    factory = AccordExecutorSimple::new;
+                    maxThreads = 1;
+                    break;
+            }
+
             for (int id = 0; id < executors.length; id++)
             {
-                AccordStateCacheMetrics metrics = new AccordStateCacheMetrics(ACCORD_STATE_CACHE);
-                AccordStateCache stateCache = new AccordStateCache(Stage.READ.executor(), Stage.MUTATION.executor(), 8 << 20, metrics);
-                executors[id] = new CommandStoreExecutor(stateCache, executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + id + ']'));
+                AccordCacheMetrics metrics = new AccordCacheMetrics(ACCORD_STATE_CACHE);
+                QueueShardModel shardModel = DatabaseDescriptor.getAccordQueueShardModel();
+                String baseName = AccordExecutor.class.getSimpleName() + '[' + id;
+                int threads = Math.min(maxThreads, Math.max(DatabaseDescriptor.getAccordConcurrentOps() / getAccordQueueShardCount(), 1));
+                switch (shardModel)
+                {
+                    case THREAD_PER_SHARD:
+                    case THREAD_PER_SHARD_SYNC_QUEUE:
+                        executors[id] = factory.get(id, shardModel == THREAD_PER_SHARD ? RUN_WITHOUT_LOCK : RUN_WITH_LOCK, 1, constant(baseName + ']'), metrics, constantFactory(Stage.READ.executor()), constantFactory(Stage.MUTATION.executor()), constantFactory(Stage.READ.executor()), agent);
+                        break;
+                    case THREAD_POOL_PER_SHARD:
+                        executors[id] = factory.get(id, RUN_WITHOUT_LOCK, threads, num -> baseName + ',' + num + ']', metrics, AccordExecutor::submitIOToSelf, AccordExecutor::submitIOToSelf, AccordExecutor::submitIOToSelf, agent);
+                        break;
+                    case THREAD_POOL_PER_SHARD_EXCLUDES_IO:
+                        executors[id] = factory.get(id, RUN_WITHOUT_LOCK, threads, num -> baseName + ',' + num + ']', metrics, constantFactory(Stage.READ.executor()), constantFactory(Stage.MUTATION.executor()), constantFactory(Stage.READ.executor()), agent);
+                        break;
+                }
             }
 
             return new AccordCommandStores(time, agent, store, random, shardDistributor, progressLogFactory, listenerFactory, journal, executors);
@@ -109,7 +146,27 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public synchronized void setCapacity(long bytes)
     {
         cacheSize = bytes;
-        refreshCacheSizes();
+        refreshCapacities();
+    }
+
+    public synchronized void setWorkingSetSize(long bytes)
+    {
+        workingSetSize = bytes;
+        refreshCapacities();
+    }
+
+    public synchronized void setCapacityAndWorkingSetSize(long newCacheSize, long newWorkingSetSize)
+    {
+        cacheSize = newCacheSize;
+        workingSetSize = newWorkingSetSize;
+        refreshCapacities();
+    }
+
+    public synchronized void setMaxQueuedLoads(int total, int range)
+    {
+        maxQueuedLoads = total;
+        maxQueuedRangeLoads = range;
+        refreshCapacities();
     }
 
     @Override
@@ -122,7 +179,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public int size()
     {
         int size = 0;
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
             size += executor.size();
         return size;
     }
@@ -131,19 +188,31 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public long weightedSize()
     {
         long size = 0;
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
             size += executor.weightedSize();
         return size;
     }
 
-    synchronized void refreshCacheSizes()
+    synchronized void refreshCapacities()
     {
-        if (count() == 0)
-            return;
-        long perExecutor = cacheSize / executors.length;
-        // TODO (low priority, safety): we might transiently breach our limit if we increase one store before decreasing another
-        for (CommandStoreExecutor executor : executors)
-            executor.execute(() -> executor.setCapacity(perExecutor));
+        long capacityPerExecutor = cacheSize / executors.length;
+        long workingSetPerExecutor = workingSetSize < 0 ? Long.MAX_VALUE : workingSetSize / executors.length;
+        int maxLoadsPerExecutor = (maxQueuedLoads + executors.length - 1) / executors.length;
+        int maxRangeLoadsPerExecutor = (maxQueuedRangeLoads + executors.length - 1) / executors.length;
+        for (AccordExecutor executor : executors)
+        {
+            executor.executeDirectlyWithLock(() -> {
+                executor.setCapacity(capacityPerExecutor);
+                executor.setWorkingSetSize(workingSetPerExecutor);
+                executor.setMaxQueuedLoads(maxLoadsPerExecutor, maxRangeLoadsPerExecutor);
+                executor.cacheExclusive().setShrinkingOn(shrinkingOn);
+            });
+        }
+    }
+
+    public List<AccordExecutor> executors()
+    {
+        return Arrays.asList(executors.clone());
     }
 
     public void waitForQuiescense()
@@ -151,23 +220,13 @@ public class AccordCommandStores extends CommandStores implements CacheSize
         boolean hadPending;
         try
         {
-            List<ExecutorPlus> executors = new ArrayList<>();
-            for (CommandStoreExecutor executor : this.executors)
-                executors.add(executor.delegate);
-
-            executors.add(READ.executor());
-            executors.add(MUTATION.executor());
-            executors.add(ACCORD_MIGRATION.executor());
-            executors.add(ACCORD_RANGE_LOADER.executor());
-
             do
             {
                 hadPending = false;
                 List<Future<?>> futures = new ArrayList<>();
-                for (ExecutorPlus executor : executors)
+                for (AccordExecutor executor : this.executors)
                 {
-                    if (!hadPending && (executor.getPendingTaskCount() > 0 || executor.getActiveTaskCount() > 0))
-                        hadPending = true;
+                    hadPending |= executor.hasTasks();
                     futures.add(executor.submit(() -> {}));
                 }
                 for (Future<?> future : futures)
@@ -190,7 +249,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize
     public synchronized void shutdown()
     {
         super.shutdown();
-        for (CommandStoreExecutor executor : executors)
+        for (AccordExecutor executor : executors)
         {
             executor.shutdown();
             try
diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java
index 19c30b0f38..9786de506b 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java
@@ -38,7 +38,6 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DataRange;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.commitlog.CommitLogPosition;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.memtable.Memtable;
@@ -68,7 +67,7 @@ public class AccordDataStore implements DataStore
     static class SnapshotBounds
     {
         final List<org.apache.cassandra.dht.Range<Token>> ranges = new ArrayList<>();
-        CommitLogPosition position;
+        long id;
     }
 
     @Override
@@ -93,7 +92,7 @@ public class AccordDataStore implements DataStore
             ColumnFamilyStore cfs = Keyspace.openAndGetStoreIfExists(tableMetadata);
             // TODO (required): when we can safely map TxnId.hlc() -> local timestamp, consult Memtable timestamps
             Memtable memtable = cfs.getCurrentMemtable();
-            e.getValue().position = memtable.getCommitLogLowerBound();
+            e.getValue().id = memtable.getMemtableId();
         }
 
         ScheduledExecutors.scheduledTasks.schedule(() -> {
@@ -106,7 +105,7 @@ public class AccordDataStore implements DataStore
                 View view = cfs.getTracker().getView();
                 for (Memtable memtable : view.getAllMemtables())
                 {
-                    if (memtable.getCommitLogLowerBound().compareTo(bounds.position) > 0) continue;
+                    if (memtable.getMemtableId() > bounds.id) continue;
                     if (!intersects(cfs, memtable, bounds.ranges)) continue;
 
                     futures.add(cfs.forceFlush(ACCORD_TXN_GC));
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutor.java b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
new file mode 100644
index 0000000000..ce778dd76a
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutor.java
@@ -0,0 +1,1025 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import accord.api.RoutingKey;
+import accord.impl.TimestampsForKey;
+import accord.local.Command;
+import accord.local.cfk.CommandsForKey;
+import accord.primitives.TxnId;
+import accord.utils.ArrayBuffers.BufferList;
+import accord.utils.IntrusivePriorityHeap;
+import accord.utils.Invariants;
+import accord.utils.QuadConsumer;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import accord.utils.TriConsumer;
+import accord.utils.TriFunction;
+import accord.utils.async.Cancellable;
+import org.agrona.collections.Object2ObjectHashMap;
+import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static org.apache.cassandra.service.accord.AccordCacheEntry.Status.EVICTED;
+import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER;
+import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER;
+import static org.apache.cassandra.service.accord.AccordCache.registerJfrListener;
+import static org.apache.cassandra.service.accord.AccordTask.State.LOADING;
+import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES;
+import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD;
+import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+public abstract class AccordExecutor implements CacheSize, AccordCacheEntry.OnLoaded, AccordCacheEntry.OnSaved, Shutdownable
+{
+    public interface AccordExecutorFactory
+    {
+        AccordExecutor get(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent);
+    }
+
+    public enum Mode { RUN_WITH_LOCK, RUN_WITHOUT_LOCK }
+
+    public interface ExecutorFunction extends BiFunction<Task, Runnable, Cancellable> {}
+    public interface ExecutorFunctionFactory extends Function<AccordExecutor, ExecutorFunction> {}
+
+    // WARNING: this is a shared object, so close is NOT idempotent
+    public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable
+    {
+        final Lock lock;
+
+        public ExclusiveGlobalCaches(Lock lock, AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand> commands, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKey, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKey)
+        {
+            super(global, commands, timestampsForKey, commandsForKey);
+            this.lock = lock;
+        }
+
+        @Override
+        public void close()
+        {
+            lock.unlock();
+        }
+    }
+
+    public static class GlobalCaches
+    {
+        public final AccordCache global;
+        public final AccordCache.Type<TxnId, Command, AccordSafeCommand> commands;
+        public final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKey;
+        public final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKey;
+
+        public GlobalCaches(AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand> commands, AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKey, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKey)
+        {
+            this.global = global;
+            this.commands = commands;
+            this.timestampsForKey = timestampsForKey;
+            this.commandsForKey = commandsForKey;
+        }
+    }
+
+    final Lock lock;
+    final Agent agent;
+    final int executorId;
+    private final AccordCache cache;
+    private final ExecutorFunction loadExecutor;
+    private final ExecutorFunction rangeLoadExecutor;
+
+    private final TaskQueue<AccordTask<?>> scanningRanges = new TaskQueue<>(SCANNING_RANGES); // never queried, just parked here while scanning
+    private final TaskQueue<AccordTask<?>> loading = new TaskQueue<>(LOADING); // never queried, just parked here while loading
+
+    private final TaskQueue<AccordTask<?>> waitingToLoadRangeTxns = new TaskQueue<>(WAITING_TO_LOAD);
+
+    private final TaskQueue<AccordTask<?>> waitingToLoad = new TaskQueue<>(WAITING_TO_LOAD);
+    private final TaskQueue<Task> waitingToRun = new TaskQueue<>(WAITING_TO_RUN);
+    private final Object2ObjectHashMap<AccordCommandStore, CommandStoreQueue> commandStoreQueues = new Object2ObjectHashMap<>();
+
+    private final AccordCacheEntry.OnLoaded onRangeLoaded = this::onRangeLoaded;
+    private final ExclusiveGlobalCaches caches;
+
+    /**
+     * The maximum total number of loads we can queue at once - this includes loads for range transactions,
+     * which are subject to this limit as well as that imposed by {@link #maxQueuedRangeLoads}
+      */
+    private int maxQueuedLoads = 64;
+    /**
+     * The maximum number of loads exclusively for range transactions we can queue at once; the {@link #maxQueuedLoads} limit also applies.
+     */
+    private int maxQueuedRangeLoads = 8;
+
+    private long maxWorkingSetSizeInBytes;
+    private long maxWorkingCapacityInBytes;
+    private int nextPosition;
+    private int activeLoads, activeRangeLoads;
+    private boolean hasPausedLoading;
+    int tasks;
+    int running;
+
+    AccordExecutor(Lock lock, int executorId, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this.lock = lock;
+        this.executorId = executorId;
+        this.cache = new AccordCache(alwaysNullTask(saveExecutor.apply(this)), this, 0, metrics);
+        this.loadExecutor = loadExecutor.apply(this);
+        this.rangeLoadExecutor = rangeLoadExecutor.apply(this);
+        this.agent = agent;
+
+        final AccordCache.Type<TxnId, Command, AccordSafeCommand> commands;
+        final AccordCache.Type<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKey;
+        final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKey;
+        commands = cache.newType(TxnId.class, COMMAND_ADAPTER);
+        registerJfrListener(executorId, commands, "Command");
+        timestampsForKey = cache.newType(RoutingKey.class,
+                                         AccordCommandStore::loadTimestampsForKey,
+                                         AccordCommandStore::saveTimestampsForKey,
+                                         Function.identity(),
+                                         AccordCommandStore::validateTimestampsForKey,
+                                         AccordObjectSizes::timestampsForKey,
+                                         AccordSafeTimestampsForKey::new);
+        registerJfrListener(executorId, timestampsForKey, "TimestampsForKey");
+        commandsForKey = cache.newType(RoutingKey.class, CFK_ADAPTER);
+        registerJfrListener(executorId, commandsForKey, "CommandsForKey");
+
+        this.caches = new ExclusiveGlobalCaches(lock, cache, commands, timestampsForKey, commandsForKey);
+        ScheduledExecutors.scheduledFastTasks.scheduleAtFixedRate(() -> {
+            executeDirectlyWithLock(cache::processNoEvictQueue);
+        }, 1L, 1L, TimeUnit.SECONDS);
+    }
+
+    public int executorId()
+    {
+        return executorId;
+    }
+
+    public ExclusiveGlobalCaches lockCaches()
+    {
+        //noinspection LockAcquiredButNotSafelyReleased
+        lock.lock();
+        return caches;
+    }
+
+    public AccordCache cacheExclusive()
+    {
+        Invariants.checkState(isOwningThread());
+        return cache;
+    }
+
+    public AccordCache cacheUnsafe()
+    {
+        return cache;
+    }
+
+    boolean hasWaitingToRun()
+    {
+        updateWaitingToRunExclusive();
+        return !waitingToRun.isEmpty();
+    }
+
+    Task pollWaitingToRunExclusive()
+    {
+        updateWaitingToRunExclusive();
+        return waitingToRun.poll();
+    }
+
+    void updateWaitingToRunExclusive()
+    {
+        maybeUnpauseLoading();
+    }
+
+    void maybeUnpauseLoading()
+    {
+        if (!hasPausedLoading)
+            return;
+
+        if (cache.weightedSize() < maxWorkingCapacityInBytes || (loading.isEmpty() && waitingToRun.isEmpty()))
+        {
+            hasPausedLoading = false;
+            enqueueLoadsExclusive();
+        }
+    }
+
+    public abstract boolean hasTasks();
+    abstract boolean isOwningThread();
+
+    private void enqueueLoadsExclusive()
+    {
+        outer: while (true)
+        {
+            TaskQueue<AccordTask<?>> queue = waitingToLoadRangeTxns.isEmpty() || activeRangeLoads >= maxQueuedRangeLoads ? waitingToLoad : waitingToLoadRangeTxns;
+            AccordTask<?> next = queue.peek();
+            if (next == null)
+                return;
+
+            if (hasPausedLoading || cache.weightedSize() >= maxWorkingCapacityInBytes)
+            {
+                // we have too much in memory already, and we have work waiting to run, so let that complete before queueing more
+                if (!loading.isEmpty() || !waitingToRun.isEmpty())
+                {
+                    hasPausedLoading = true;
+                    return;
+                }
+            }
+
+            switch (next.state())
+            {
+                default:
+                {
+                    failExclusive(next, new AssertionError("Unexpected state: " + next.toDescription()));
+                    break;
+                }
+                case WAITING_TO_SCAN_RANGES:
+                    if (activeRangeLoads >= maxQueuedRangeLoads)
+                    {
+                        parkRangeLoad(next);
+                    }
+                    else
+                    {
+                        ++activeRangeLoads;
+                        ++activeLoads;
+                        next.rangeScanner().start(rangeLoadExecutor);
+                        updateQueue(next);
+                    }
+                    break;
+
+                case WAITING_TO_LOAD:
+                    while (true)
+                    {
+                        AccordCacheEntry<?, ?> load = next.peekWaitingToLoad();
+                        boolean isForRange = isForRange(next, load);
+                        if (isForRange && activeRangeLoads >= maxQueuedRangeLoads)
+                        {
+                            parkRangeLoad(next);
+                            continue outer;
+                        }
+
+                        Invariants.checkState(load != null);
+                        AccordCacheEntry.OnLoaded onLoaded = this;
+                        ++activeLoads;
+                        if (isForRange)
+                        {
+                            ++activeRangeLoads;
+                            onLoaded = onRangeLoaded;
+                        }
+
+                        for (AccordTask<?> task : cache.load(loadExecutor, next, load, onLoaded))
+                        {
+                            if (task == next) continue;
+                            if (task.onLoading(load))
+                                updateQueue(task);
+                        }
+                        Object prev = next.pollWaitingToLoad();
+                        Invariants.checkState(prev == load);
+                        if (next.peekWaitingToLoad() == null)
+                            break;
+
+                        Invariants.checkState(next.state() == WAITING_TO_LOAD, "Invalid state: %s", next);
+                        if (activeLoads >= maxQueuedLoads)
+                            return;
+                    }
+                    Invariants.checkState(next.state().compareTo(LOADING) >= 0, "Invalid state: %s", next);
+                    updateQueue(next);
+            }
+        }
+    }
+
+    private boolean isForRange(AccordTask<?> task, AccordCacheEntry<?, ?> load)
+    {
+        boolean isForRangeTxn = task.hasRanges();
+        if (!isForRangeTxn)
+            return false;
+
+        for (AccordTask<?> t : load.loadingOrWaiting().waiters())
+        {
+            if (!t.hasRanges())
+                return false;
+        }
+        return true;
+    }
+
+    private void parkRangeLoad(AccordTask<?> task)
+    {
+        if (task.queued() != waitingToLoadRangeTxns)
+        {
+            task.unqueueIfQueued();
+            task.addToQueue(waitingToLoadRangeTxns);
+        }
+    }
+
+    void consumeExclusive(Object object)
+    {
+        try
+        {
+            if (object instanceof AccordTask<?>)
+                loadExclusive((AccordTask<?>) object);
+            else
+                ((SubmitAsync) object).acceptExclusive(this);
+        }
+        catch (Throwable t)
+        {
+            agent.onUncaughtException(t);
+        }
+    }
+
+    private void updateQueue(AccordTask<?> task)
+    {
+        task.unqueueIfQueued();
+        switch (task.state())
+        {
+            default: throw new AssertionError("Unexpected state: " + task.toDescription());
+            case WAITING_TO_SCAN_RANGES:
+            case WAITING_TO_LOAD:
+                task.addToQueue(waitingToLoad);
+                break;
+            case SCANNING_RANGES:
+                task.addToQueue(scanningRanges);
+                break;
+            case LOADING:
+                task.addToQueue(loading);
+                break;
+            case WAITING_TO_RUN:
+                task.runQueuedAt = nanoTime();
+                commandStoreQueues.computeIfAbsent(task.commandStore, CommandStoreQueue::new)
+                                  .append(task);
+                break;
+        }
+    }
+    
+    private void waitingToRun(Task task)
+    {
+        if (task.commandStore == null)
+        {
+            waitingToRun.append(task);
+        }
+        else
+        {
+            commandStoreQueues.computeIfAbsent(task.commandStore, CommandStoreQueue::new)
+                              .append(task);
+        }
+    }
+
+    private Cancellable submitIOExclusive(Task parent, Runnable run)
+    {
+        Invariants.checkState(isOwningThread());
+        ++tasks;
+        PlainRunnable task = new PlainRunnable(null, run, null);
+        // TODO (expected): adopt queue position of the submitting task
+        if (parent == null) assignNewQueuePosition(task);
+        else assignQueueSubPosition(parent, task);
+        waitingToRun.append(task);
+        return task;
+    }
+
+    private void assignNewQueuePosition(Task task)
+    {
+        task.queuePosition = (((long)++nextPosition) & 0xffffffffL) << 31;
+    }
+
+    private void assignQueueSubPosition(Task parent, Task task)
+    {
+        task.queuePosition = parent.queuePosition | (++nextPosition & 0x7fffffff);
+    }
+
+    public Executor executor(AccordCommandStore commandStore)
+    {
+        return task -> AccordExecutor.this.submit(task, commandStore);
+    }
+
+    public <R> void submit(AccordTask<R> operation)
+    {
+        submit(AccordExecutor::loadExclusive, Function.identity(), operation);
+    }
+
+    public <R> void cancel(AccordTask<R> operation)
+    {
+        submit(AccordExecutor::cancelExclusive, OnCancel::new, operation);
+    }
+
+    public void onScannedRanges(AccordTask<?> task, Throwable fail)
+    {
+        submit(AccordExecutor::onScannedRangesExclusive, OnScannedRanges::new, task, fail);
+    }
+
+    public <K, V> void onSaved(AccordCacheEntry<K, V> saved, Object identity, Throwable fail)
+    {
+        submit(AccordExecutor::onSavedExclusive, OnSaved::new, saved, identity, fail);
+    }
+
+    @Override
+    public <K, V> void onLoaded(AccordCacheEntry<K, V> loaded, V value, Throwable fail)
+    {
+        submit(AccordExecutor::onLoadedExclusive, OnLoaded::new, loaded, value, fail, false);
+    }
+
+    public <K, V> void onRangeLoaded(AccordCacheEntry<K, V> loaded, V value, Throwable fail)
+    {
+        submit(AccordExecutor::onLoadedExclusive, OnLoaded::new, loaded, value, fail, true);
+    }
+
+    private <P1> void submit(BiConsumer<AccordExecutor, P1> sync, Function<P1, ?> async, P1 p1)
+    {
+        submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a), (f, p1a, p2a, p3) -> f.apply(p1a), sync, async, p1, null, null);
+    }
+
+    private <P1, P2> void submit(TriConsumer<AccordExecutor, P1, P2> sync, BiFunction<P1, P2, ?> async, P1 p1, P2 p2)
+    {
+        submit((e, c, p1a, p2a, p3) -> c.accept(e, p1a, p2a), (f, p1a, p2a, p3) -> f.apply(p1a, p2a), sync, async, p1, p2, null);
+    }
+
+    private <P1, P2, P3> void submit(QuadConsumer<AccordExecutor, P1, P2, P3> sync, TriFunction<P1, P2, P3, ?> async, P1 p1, P2 p2, P3 p3)
+    {
+        submit((e, c, p1a, p2a, p3a) -> c.accept(e, p1a, p2a, p3a), TriFunction::apply, sync, async, p1, p2, p3);
+    }
+
+    private <P1, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1, P2, P3, P4> sync, QuadFunction<P1, P2, P3, P4, Object> async, P1 p1, P2 p2, P3 p3, P4 p4)
+    {
+        submit(sync, async, p1, p1, p2, p3, p4);
+    }
+
+    abstract <P1s, P1a, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4);
+
+    private void submitExclusive(AsyncPromise<Void> result, Runnable run, AccordCommandStore commandStore)
+    {
+        ++tasks;
+        PlainRunnable task = new PlainRunnable(result, run, commandStore);
+        task.queuePosition = ++nextPosition;
+        waitingToRun(task);
+    }
+
+    private void submitExclusive(AsyncPromise<Void> result, PlainRunnable task)
+    {
+        ++tasks;
+        task.queuePosition = ++nextPosition;
+        waitingToRun(task);
+    }
+
+    private void loadExclusive(AccordTask<?> task)
+    {
+        ++tasks;
+        assignNewQueuePosition(task);
+        task.setupExclusive();
+        updateQueue(task);
+        enqueueLoadsExclusive();
+    }
+
+    private void cancelExclusive(AccordTask<?> task)
+    {
+        switch (task.state())
+        {
+            default:
+            case INITIALIZED:
+                // we could be cancelled before we even reach the queue
+                task.cancelExclusive();
+                break;
+
+            case LOADING:
+            case WAITING_TO_LOAD:
+            case WAITING_TO_SCAN_RANGES:
+            case SCANNING_RANGES:
+            case WAITING_TO_RUN:
+                --tasks;
+                task.unqueueIfQueued();
+                task.cancelExclusive();
+                break;
+
+            case RUNNING:
+            case PERSISTING:
+            case FINISHED:
+            case CANCELLED:
+            case FAILED:
+                // cannot safely cancel
+        }
+    }
+
+    private void onScannedRangesExclusive(AccordTask<?> task, Throwable fail)
+    {
+        --activeLoads;
+        --activeRangeLoads;
+        // the task may have already been cancelled, in which case we don't need to fail it
+        if (!task.state().isExecuted())
+        {
+            if (fail != null)
+            {
+                failExclusive(task, fail);
+            }
+            else
+            {
+                task.rangeScanner().scannedExclusive();
+                updateQueue(task);
+            }
+        }
+        enqueueLoadsExclusive();
+    }
+
+    private void failExclusive(AccordTask<?> task, Throwable fail)
+    {
+        if (task.state().isExecuted())
+            return;
+
+        --tasks;
+        try { task.failExclusive(fail); }
+        catch (Throwable t) { agent.onUncaughtException(t); }
+        finally
+        {
+            task.unqueueIfQueued();
+            task.cleanupExclusive();
+        }
+    }
+
+    private <K, V> void onSavedExclusive(AccordCacheEntry<K, V> state, Object identity, Throwable fail)
+    {
+        cache.saved(state, identity, fail);
+    }
+
+    private <K, V> void onLoadedExclusive(AccordCacheEntry<K, V> loaded, V value, Throwable fail, boolean isForRange)
+    {
+        --activeLoads;
+        --activeRangeLoads;
+
+        if (loaded.status() != EVICTED)
+        {
+            try (BufferList<AccordTask<?>> tasks = loaded.loading().copyWaiters())
+            {
+                if (fail != null)
+                {
+                    for (AccordTask<?> task : tasks)
+                        failExclusive(task, fail);
+                    cache.failedToLoad(loaded);
+                }
+                else
+                {
+                    cache.loaded(loaded, value);
+                    for (AccordTask<?> task : tasks)
+                    {
+                        if (task.onLoad(loaded))
+                        {
+                            Invariants.checkState(task.queued() == loading);
+                            task.unqueue();
+                            waitingToRun(task);
+                        }
+                    }
+                }
+            }
+        }
+
+        enqueueLoadsExclusive();
+    }
+
+    public Future<?> submit(Runnable run)
+    {
+        return submit(run, null);
+    }
+
+    // TODO (expected): offer queue jumping/priorities
+    public Future<?> submit(Runnable run, AccordCommandStore commandStore)
+    {
+        PlainRunnable task = new PlainRunnable(new AsyncPromise<>(), run, commandStore);
+        AsyncPromise<Void> result = new AsyncPromise<>();
+        submit(AccordExecutor::submitExclusive, SubmitPlainRunnable::new, result, run, commandStore);
+        return result;
+    }
+
+    public void execute(Runnable command)
+    {
+        submit(command);
+    }
+
+    public void executeDirectlyWithLock(Runnable command)
+    {
+        lock.lock();
+        try
+        {
+            command.run();
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    public void execute(Runnable command, AccordCommandStore commandStore)
+    {
+        submit(command, commandStore);
+    }
+
+    @Override
+    public void setCapacity(long bytes)
+    {
+        Invariants.checkState(isOwningThread());
+        cache.setCapacity(bytes);
+        maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes;
+    }
+
+    public void setWorkingSetSize(long bytes)
+    {
+        Invariants.checkState(isOwningThread());
+        maxWorkingSetSizeInBytes = bytes;
+        maxWorkingCapacityInBytes = cache.capacity() + maxWorkingSetSizeInBytes;
+        if (maxWorkingCapacityInBytes < maxWorkingSetSizeInBytes)
+            maxWorkingCapacityInBytes = Long.MAX_VALUE;
+    }
+
+    public void setMaxQueuedLoads(int total, int range)
+    {
+        Invariants.checkState(isOwningThread());
+        maxQueuedLoads = total;
+        maxQueuedRangeLoads = range;
+    }
+
+    @Override
+    public long capacity()
+    {
+        return cache.capacity();
+    }
+
+    @Override
+    public int size()
+    {
+        return cache.size();
+    }
+
+    @Override
+    public long weightedSize()
+    {
+        return cache.weightedSize();
+    }
+
+    public static abstract class Task extends IntrusivePriorityHeap.Node
+    {
+        final AccordCommandStore commandStore;
+        long queuePosition;
+
+        protected Task(AccordCommandStore commandStore)
+        {
+            this.commandStore = commandStore;
+        }
+
+        /**
+         * Prepare to run while holding the state cache lock
+         */
+        abstract protected void preRunExclusive();
+
+        /**
+         * Run the command; the state cache lock may or may not be held depending on the executor implementation
+         */
+        abstract protected void run();
+        /**
+         * Fail the command; the state cache lock may or may not be held depending on the executor implementation
+         */
+        abstract protected void fail(Throwable fail);
+
+        /**
+         * Cleanup the command while holding the state cache lock
+         */
+        abstract protected void cleanupExclusive();
+
+        abstract protected void addToQueue(TaskQueue queue);
+    }
+
+    class CommandStoreQueue extends Task
+    {
+        final TaskQueue<Task> queue = new TaskQueue<>(WAITING_TO_RUN);
+        Task next;
+
+        CommandStoreQueue(AccordCommandStore commandStore)
+        {
+            super(commandStore);
+        }
+
+        @Override
+        protected void preRunExclusive()
+        {
+            Invariants.checkState(next != null);
+            Thread self = Thread.currentThread();
+            commandStore.setOwner(self, self);
+            next.preRunExclusive();
+        }
+
+        @Override
+        protected void run()
+        {
+            next.run();
+        }
+
+        @Override
+        protected void fail(Throwable t)
+        {
+            next.fail(t);
+        }
+
+        @Override
+        protected void cleanupExclusive()
+        {
+            next.cleanupExclusive();
+            commandStore.setOwner(null, Thread.currentThread());
+            updateNext(queue.poll());
+        }
+
+        @Override
+        protected void addToQueue(TaskQueue queue)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        void append(Task task)
+        {   // TODO (expected): if the new task is higher priority, replace next
+            if (next == null) updateNext(task);
+            else task.addToQueue(queue);
+        }
+
+        void updateNext(Task task)
+        {
+            next = task;
+            if (task != null)
+            {
+                queuePosition = task.queuePosition;
+                waitingToRun.append(this);
+            }
+        }
+    }
+
+    static final class TaskQueue<T extends Task> extends IntrusivePriorityHeap<T>
+    {
+        final AccordTask.State kind;
+
+        TaskQueue(AccordTask.State kind)
+        {
+            this.kind = kind;
+        }
+
+        @Override
+        public int compare(T o1, T o2)
+        {
+            return Long.compare(o1.queuePosition, o2.queuePosition);
+        }
+        public void append(T task)
+        {
+            super.append(task);
+        }
+
+        public T poll()
+        {
+            ensureHeapified();
+            return pollNode();
+        }
+
+        public T peek()
+        {
+            ensureHeapified();
+            return peekNode();
+        }
+
+        public void remove(T remove)
+        {
+            super.remove(remove);
+        }
+
+        public boolean contains(T contains)
+        {
+            return super.contains(contains);
+        }
+    }
+
+    private abstract static class SubmitAsync
+    {
+        abstract void acceptExclusive(AccordExecutor executor);
+    }
+
+    private static class SubmitPlainRunnable extends SubmitAsync
+    {
+        final AsyncPromise<Void> result;
+        final Runnable run;
+        final AccordCommandStore commandStore;
+
+        private SubmitPlainRunnable(AsyncPromise<Void> result, Runnable run, AccordCommandStore commandStore)
+        {
+            this.result = result;
+            this.run = run;
+            this.commandStore = commandStore;
+        }
+
+        @Override
+        void acceptExclusive(AccordExecutor executor)
+        {
+            executor.submitExclusive(result, run, commandStore);
+        }
+    }
+
+    private static class OnLoaded<K, V> extends SubmitAsync
+    {
+        static final int FAIL = 1;
+        static final int RANGE = 2;
+        final AccordCacheEntry<K, V> loaded;
+        final Object result;
+        final int flags;
+
+        OnLoaded(AccordCacheEntry<K, V> loaded, V success, Throwable fail, boolean isForRange)
+        {
+            this.loaded = loaded;
+            int flags = isForRange ? RANGE : 0;
+            if (fail == null)
+            {
+                result = success;
+            }
+            else
+            {
+                result = fail;
+                flags |= FAIL;
+            }
+            this.flags = flags;
+        }
+
+        V success()
+        {
+            return (flags & FAIL) == 0 ? (V) result : null;
+        }
+
+        Throwable fail()
+        {
+            return (flags & FAIL) == 0 ? null : (Throwable) result;
+        }
+
+        boolean isForRange()
+        {
+            return (flags & RANGE) != 0;
+        }
+
+        @Override
+        void acceptExclusive(AccordExecutor executor)
+        {
+            executor.onLoadedExclusive(loaded, success(), fail(), isForRange());
+        }
+    }
+
+    private static class OnScannedRanges extends SubmitAsync
+    {
+        final AccordTask<?> scanned;
+        final Throwable fail;
+
+        private OnScannedRanges(AccordTask<?> scanned, Throwable fail)
+        {
+            this.scanned = scanned;
+            this.fail = fail;
+        }
+
+        @Override
+        void acceptExclusive(AccordExecutor executor)
+        {
+            executor.onScannedRangesExclusive(scanned, fail);
+        }
+    }
+
+    private static class OnSaved<K, V> extends SubmitAsync
+    {
+        final AccordCacheEntry<K, V> state;
+        final Object identity;
+        final Throwable fail;
+
+        private OnSaved(AccordCacheEntry<K, V> state, Object identity, Throwable fail)
+        {
+            this.state = state;
+            this.identity = identity;
+            this.fail = fail;
+        }
+
+        @Override
+        void acceptExclusive(AccordExecutor executor)
+        {
+            executor.onSavedExclusive(state, identity, fail);
+        }
+    }
+
+    private static class OnCancel<R> extends SubmitAsync
+    {
+        final AccordTask<R> cancel;
+
+        private OnCancel(AccordTask<R> cancel)
+        {
+            this.cancel = cancel;
+        }
+
+        @Override
+        void acceptExclusive(AccordExecutor executor)
+        {
+            executor.cancelExclusive(cancel);
+        }
+    }
+
+    static <O> IntFunction<O> constant(O out)
+    {
+        return ignore -> out;
+    }
+
+    static ExecutorFunctionFactory constantFactory(ExecutorFunction exec)
+    {
+        return ignore -> exec;
+    }
+
+    static ExecutorFunctionFactory constantFactory(ExecutorPlus exec)
+    {
+        return ignore -> wrap(exec);
+    }
+
+    static ExecutorFunction wrap(ExecutorPlus exec)
+    {
+        return (t, r) -> wrap(exec.submit(r));
+    }
+
+    static Cancellable wrap(Future<?> f)
+    {
+        return () -> f.cancel(false);
+    }
+
+    public static ExecutorFunction submitIOToSelf(AccordExecutor executor)
+    {
+        return executor::submitIOExclusive;
+    }
+
+    private static Function<Runnable, Cancellable> alwaysNullTask(ExecutorFunction f)
+    {
+        return r -> f.apply(null, r);
+    }
+
+    class PlainRunnable extends Task implements Cancellable
+    {   // TODO (expected): support cancellation
+        final AsyncPromise<Void> result;
+        final Runnable run;
+
+        PlainRunnable(AsyncPromise<Void> result, Runnable run, AccordCommandStore commandStore)
+        {
+            super(commandStore);
+            this.result = result;
+            this.run = run;
+        }
+
+        @Override
+        protected void preRunExclusive() {}
+
+        @Override
+        protected void run()
+        {
+            run.run();
+            if (result != null)
+                result.trySuccess(null);
+        }
+
+        @Override
+        protected void fail(Throwable t)
+        {
+            if (result != null)
+                result.tryFailure(t);
+            agent.onUncaughtException(t);
+        }
+
+        @Override
+        protected void cleanupExclusive() {}
+
+        @Override
+        protected void addToQueue(TaskQueue queue)
+        {
+            Invariants.checkState(queue.kind == WAITING_TO_RUN);
+            queue.append(this);
+        }
+
+        @Override
+        public void cancel()
+        {
+            executeDirectlyWithLock(() -> {
+                if (isInHeap())
+                {
+                    waitingToRun.remove(this);
+                    if (result != null)
+                        result.cancel(false);
+                }
+            });
+        }
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java
new file mode 100644
index 0000000000..176470f113
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.locks.Lock;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.utils.concurrent.ConcurrentLinkedStack;
+
+import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+
+abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
+{
+    final ConcurrentLinkedStack<Object> submitted = new ConcurrentLinkedStack<>();
+    boolean isHeldByExecutor;
+
+    AccordExecutorAbstractLockLoop(Lock lock, int executorId, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    abstract void notifyWorkExclusive();
+    abstract void awaitExclusive() throws InterruptedException;
+    abstract boolean isInLoop();
+    abstract <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4);
+
+    <P1s, P1a, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        // if we're a loop thread, we will poll the waitingToRun queue when we come around
+        if (isInLoop()) submitted.push(async.apply(p1a, p2, p3, p4));
+        else submitExternal(sync, async, p1s, p1a, p2, p3, p4);
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternalExclusive(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        try
+        {
+            try
+            {
+                drainSubmittedExclusive();
+            }
+            catch (Throwable t)
+            {
+                try { sync.accept(this, p1s, p2, p3, p4); }
+                catch (Throwable t2) { t.addSuppressed(t2); }
+                throw t;
+            }
+            sync.accept(this, p1s, p2, p3, p4);
+        }
+        finally
+        {
+            notifyIfMoreWorkExclusive();
+        }
+    }
+
+    public boolean hasTasks()
+    {
+        if (tasks > 0 || !submitted.isEmpty() || running > 0)
+            return true;
+
+        lock.lock();
+        try
+        {
+            return tasks > 0 || !submitted.isEmpty() || running > 0;
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    void updateWaitingToRunExclusive()
+    {
+        drainSubmittedExclusive();
+        super.updateWaitingToRunExclusive();
+    }
+
+    void drainSubmittedExclusive()
+    {
+        submitted.drain(AccordExecutor::consumeExclusive, this, true);
+    }
+
+    void notifyIfMoreWorkExclusive()
+    {
+        if (hasWaitingToRun())
+            notifyWorkExclusive();
+    }
+
+    private void enterLockExclusive()
+    {
+        isHeldByExecutor = true;
+    }
+
+    private void exitLockExclusive()
+    {
+        isHeldByExecutor = false;
+        notifyIfMoreWorkExclusive();
+    }
+
+    private void pauseExclusive()
+    {
+        --running;
+    }
+
+    private void resumeExclusive()
+    {
+        ++running;
+    }
+
+    Interruptible.Task task(Mode mode)
+    {
+        return mode == RUN_WITH_LOCK ? this::runWithLock : this::runWithoutLock;
+    }
+
+    protected void runWithLock(Interruptible.State state) throws InterruptedException
+    {
+        lock.lockInterruptibly();
+        try
+        {
+            resumeExclusive();
+            enterLockExclusive();
+            while (true)
+            {
+                Task task = pollWaitingToRunExclusive();
+
+                if (task != null)
+                {
+                    --tasks;
+                    try
+                    {
+                        task.preRunExclusive();
+                        task.run();
+                    }
+                    catch (Throwable t)
+                    {
+                        task.fail(t);
+                    }
+                    finally
+                    {
+                        task.cleanupExclusive();
+                    }
+                }
+                else
+                {
+                    if (state != NORMAL)
+                    {
+                        pauseExclusive();
+                        exitLockExclusive();
+                        return;
+                    }
+
+                    pauseExclusive();
+                    awaitExclusive();
+                    resumeExclusive();
+                }
+            }
+        }
+        catch (Throwable t)
+        {
+            pauseExclusive();
+            exitLockExclusive();
+            throw t;
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    protected void runWithoutLock(Interruptible.State state) throws InterruptedException
+    {
+        Task task = null;
+        while (true)
+        {
+            lock.lock();
+            try
+            {
+                if (task != null) task.cleanupExclusive();
+                else resumeExclusive();
+                enterLockExclusive();
+
+                while (true)
+                {
+                    task = pollWaitingToRunExclusive();
+                    if (task != null)
+                    {
+                        exitLockExclusive();
+                        break;
+                    }
+
+                    if (state != NORMAL)
+                    {
+                        exitLockExclusive();
+                        return;
+                    }
+
+                    pauseExclusive();
+                    awaitExclusive();
+                    resumeExclusive();
+                }
+                --tasks;
+                task.preRunExclusive();
+            }
+            catch (Throwable t)
+            {
+                if (task != null)
+                {
+                    try { task.fail(t); }
+                    catch (Throwable t2) { t.addSuppressed(t2); }
+                    try { task.cleanupExclusive(); }
+                    catch (Throwable t2) { t.addSuppressed(t2); }
+                    try { agent.onUncaughtException(t); }
+                    catch (Throwable t2) { /* nothing we can sensibly do after already reporting */ }
+                }
+                pauseExclusive();
+                exitLockExclusive();
+                throw t;
+            }
+            finally
+            {
+                lock.unlock();
+            }
+
+            try
+            {
+                task.run();
+            }
+            catch (Throwable t)
+            {
+                try { task.fail(t); }
+                catch (Throwable t2)
+                {
+                    try
+                    {
+                        t2.addSuppressed(t);
+                        agent.onUncaughtException(t2);
+                    }
+                    catch (Throwable t3)
+                    {
+                        // empty to ensure we definitely loop so we cleanup the task
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java
new file mode 100644
index 0000000000..a216e60ae8
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.locks.Lock;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+
+abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstractLockLoop
+{
+    AccordExecutorAbstractSemiSyncSubmit(Lock lock, int executorId, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    abstract void notifyWorkAsync();
+    abstract void awaitExclusive() throws InterruptedException;
+
+    Interruptible.Task task(Mode mode)
+    {
+        return mode == RUN_WITH_LOCK ? this::runWithLock : this::runWithoutLock;
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        if (!lock.tryLock())
+        {
+            submitted.push(async.apply(p1a, p2, p3, p4));
+            notifyWorkAsync();
+            return;
+        }
+
+        try
+        {
+            submitExternalExclusive(sync, async, p1s, p1a, p2, p3, p4);
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java
new file mode 100644
index 0000000000..c0f0c5c06a
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorAsyncSubmit.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.utils.concurrent.LockWithAsyncSignal;
+
+// WARNING: experimental - needs more testing
+class AccordExecutorAsyncSubmit extends AccordExecutorAbstractSemiSyncSubmit
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final LockWithAsyncSignal lock;
+
+    public AccordExecutorAsyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new LockWithAsyncSignal(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorAsyncSubmit(LockWithAsyncSignal lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        lock.clearSignal();
+        if (submitted.isEmpty())
+            lock.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    void notifyWorkAsync()
+    {
+        lock.signal();
+    }
+
+    @Override
+    void notifyWorkExclusive()
+    {
+        lock.signal();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isOwner(Thread.currentThread());
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java
new file mode 100644
index 0000000000..5a19d9aa22
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorInfiniteLoops.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.IntFunction;
+
+import accord.utils.Invariants;
+import org.agrona.collections.LongHashSet;
+import org.apache.cassandra.concurrent.InfiniteLoopExecutor;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.service.accord.AccordExecutor.Mode;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+class AccordExecutorInfiniteLoops implements Shutdownable
+{
+    private final Interruptible[] loops;
+    private final LongHashSet threadIds;
+
+    public AccordExecutorInfiniteLoops(Mode mode, int threads, IntFunction<String> name, Function<Mode, Interruptible.Task> tasks)
+    {
+        Invariants.checkState(mode == RUN_WITH_LOCK ? threads == 1 : threads >= 1);
+        final LongHashSet threadIds = new LongHashSet(threads, 0.5f);
+        this.loops = new Interruptible[threads];
+        for (int i = 0; i < threads; ++i)
+        {
+            loops[i] = executorFactory().infiniteLoop(name.apply(i), tasks.apply(mode), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+            if (loops[i] instanceof InfiniteLoopExecutor)
+                threadIds.add(((InfiniteLoopExecutor) loops[i]).threadId());
+        }
+        this.threadIds = threadIds;
+    }
+
+    public boolean isInLoop()
+    {
+        return threadIds.contains(Thread.currentThread().getId());
+    }
+
+    @Override
+    public void shutdown()
+    {
+        for (Interruptible loop : loops)
+            loop.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        for (Interruptible loop : loops)
+            loop.shutdownNow();
+        return null;
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        for (Interruptible loop : loops)
+        {
+            if (!loop.isTerminated())
+                return false;
+        }
+        return true;
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        long deadline = nanoTime() + unit.toNanos(timeout);
+        for (Interruptible loop : loops)
+        {
+            long wait = deadline - nanoTime();
+            if (!loop.awaitTermination(wait, unit))
+                return false;
+        }
+        return true;
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java
new file mode 100644
index 0000000000..3272d52971
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSemiSyncSubmit.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+// WARNING: experimental - needs more testing
+class AccordExecutorSemiSyncSubmit extends AccordExecutorAbstractSemiSyncSubmit
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final ReentrantLock lock;
+    private final Condition hasWork;
+
+    public AccordExecutorSemiSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSemiSyncSubmit(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.hasWork = lock.newCondition();
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        if (submitted.isEmpty())
+            hasWork.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    void notifyWorkAsync()
+    {
+        // we check running both sides of tryLock for ordering guarantees
+        boolean hadRunning = isHeldByExecutor;
+        if (lock.tryLock())
+        {
+            try { hasWork.signal(); }
+            finally { lock.unlock(); }
+        }
+        else if (!hadRunning || !isHeldByExecutor)
+        {
+            lock.lock();
+            try { hasWork.signal(); }
+            finally { lock.unlock(); }
+        }
+    }
+    
+    @Override
+    void notifyWorkExclusive()
+    {
+        hasWork.signal();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java
new file mode 100644
index 0000000000..3e8e76eb98
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSimple.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import accord.utils.Invariants;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+class AccordExecutorSimple extends AccordExecutor
+{
+    final ExecutorPlus executor;
+    final ReentrantLock lock;
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, Agent agent)
+    {
+        this(executorId, name, metrics, Stage.READ.executor(), Stage.MUTATION.executor(), Stage.READ.executor(), agent);
+    }
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, name, metrics, wrap(loadExecutor), wrap(saveExecutor), wrap(rangeLoadExecutor), agent);
+    }
+
+    public AccordExecutorSimple(int executorId, String name, AccordCacheMetrics metrics, ExecutorFunction loadExecutor, ExecutorFunction saveExecutor, ExecutorFunction rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSimple(ReentrantLock lock, int executorId, String name, AccordCacheMetrics metrics, ExecutorFunction loadExecutor, ExecutorFunction saveExecutor, ExecutorFunction rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, constantFactory(loadExecutor), constantFactory(saveExecutor), constantFactory(rangeLoadExecutor), agent);
+        this.lock = lock;
+        this.executor = executorFactory().sequential(name);
+    }
+
+    public AccordExecutorSimple(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    public AccordExecutorSimple(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        Invariants.checkArgument(threads == 1);
+        this.lock = lock;
+        this.executor = executorFactory().sequential(name.apply(0));
+
+    }
+
+    @Override
+    public boolean hasTasks()
+    {
+        return tasks + executor.getActiveTaskCount() + executor.getPendingTaskCount() > 0;
+    }
+
+    protected void run()
+    {
+        lock.lock();
+        try
+        {
+            running = 1;
+            while (true)
+            {
+                Task task = pollWaitingToRunExclusive();
+                if (task == null)
+                    return;
+
+                --tasks;
+                try { task.preRunExclusive(); task.run(); }
+                catch (Throwable t) { task.fail(t); }
+                finally { task.cleanupExclusive(); }
+            }
+        }
+        catch (Throwable t)
+        {
+            throw t;
+        }
+        finally
+        {
+            running = 0;
+            if (hasWaitingToRun())
+                executor.execute(this::run);
+            lock.unlock();
+        }
+    }
+
+    @Override
+    <P1s, P1a, P2, P3, P4> void submit(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        lock.lock();
+        try
+        {
+            sync.accept(this, p1s, p2, p3, p4);
+        }
+        finally
+        {
+            if (hasWaitingToRun())
+                executor.execute(this::run);
+
+            lock.unlock();
+        }
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return executor.isTerminated();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        executor.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return executor.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    {
+        return executor.awaitTermination(timeout, units);
+    }
+
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java b/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java
new file mode 100644
index 0000000000..2ae7d36ad5
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordExecutorSyncSubmit.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
+
+import accord.api.Agent;
+import accord.utils.QuadFunction;
+import accord.utils.QuintConsumer;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+
+class AccordExecutorSyncSubmit extends AccordExecutorAbstractLockLoop
+{
+    private final AccordExecutorInfiniteLoops loops;
+    private final ReentrantLock lock;
+    private final Condition hasWork;
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, String name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, mode, 1, constant(name), metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, ExecutorPlus rangeLoadExecutor, Agent agent)
+    {
+        this(executorId, mode, threads, name, metrics, constantFactory(loadExecutor), constantFactory(saveExecutor), constantFactory(rangeLoadExecutor), agent);
+    }
+
+    public AccordExecutorSyncSubmit(int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        this(new ReentrantLock(), executorId, mode, threads, name, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+    }
+
+    private AccordExecutorSyncSubmit(ReentrantLock lock, int executorId, Mode mode, int threads, IntFunction<String> name, AccordCacheMetrics metrics, ExecutorFunctionFactory loadExecutor, ExecutorFunctionFactory saveExecutor, ExecutorFunctionFactory rangeLoadExecutor, Agent agent)
+    {
+        super(lock, executorId, metrics, loadExecutor, saveExecutor, rangeLoadExecutor, agent);
+        this.lock = lock;
+        this.hasWork = lock.newCondition();
+        this.loops = new AccordExecutorInfiniteLoops(mode, threads, name, this::task);
+    }
+
+    @Override
+    void awaitExclusive() throws InterruptedException
+    {
+        hasWork.await();
+    }
+
+    @Override
+    boolean isInLoop()
+    {
+        return loops.isInLoop();
+    }
+
+    @Override
+    boolean isOwningThread()
+    {
+        return lock.isHeldByCurrentThread();
+    }
+
+    @Override
+    void notifyWorkExclusive()
+    {
+        hasWork.signal();
+    }
+
+    <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Object> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
+    {
+        lock.lock();
+        try
+        {
+            submitExternalExclusive(sync, async, p1s, p1a, p2, p3, p4);
+        }
+        finally
+        {
+            lock.unlock();
+        }
+    }
+
+    @Override
+    public void shutdown()
+    {
+        loops.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return loops.shutdownNow();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return loops.isTerminated();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException
+    {
+        return loops.awaitTermination(timeout, unit);
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java
index 61cb7a4e28..8eb5eeb2ad 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java
@@ -203,7 +203,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
             future.addCallback((state, fail) -> {
                 if (fail == null) success(from, Ranges.of(range));
                 else fail(from, Ranges.of(range), fail);
-            }, ((AccordCommandStore) commandStore()).executor());
+            }, ((AccordCommandStore) commandStore()).taskExecutor());
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
index aa79a492a9..3b6bd4c916 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java
@@ -62,6 +62,7 @@ import org.apache.cassandra.journal.ValueSerializer;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.accord.AccordJournalValueSerializers.IdentityAccumulator;
 import org.apache.cassandra.service.accord.JournalKey.JournalKeySupport;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.utils.ExecutorUtils;
 
 import static accord.primitives.SaveStatus.ErasedOrVestigial;
@@ -86,14 +87,16 @@ public class AccordJournal implements IJournal, Shutdownable
     private final Journal<JournalKey, Object> journal;
     private final AccordJournalTable<JournalKey, Object> journalTable;
     private final Params params;
+    private final AccordAgent agent;
     Node node;
 
     enum Status { INITIALIZED, STARTING, REPLAY, STARTED, TERMINATING, TERMINATED }
     private volatile Status status = Status.INITIALIZED;
 
     @VisibleForTesting
-    public AccordJournal(Params params)
+    public AccordJournal(Params params, AccordAgent agent)
     {
+        this.agent = agent;
         File directory = new File(DatabaseDescriptor.getAccordJournalDirectory());
         this.journal = new Journal<>("AccordJournal", directory, params, JournalKey.SUPPORT,
                                      // In Accord, we are using streaming serialization, i.e. Reader/Writer interfaces instead of materializing objects
@@ -180,7 +183,7 @@ public class AccordJournal implements IJournal, Shutdownable
     public Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
     {
         SavedCommand.Builder builder = loadDiffs(commandStoreId, txnId);
-        Cleanup cleanup = builder.shouldCleanup(redundantBefore, durableBefore);
+        Cleanup cleanup = builder.shouldCleanup(agent, redundantBefore, durableBefore);
         switch (cleanup)
         {
             case EXPUNGE_PARTIAL:
@@ -195,7 +198,10 @@ public class AccordJournal implements IJournal, Shutdownable
     public SavedCommand.MinimalCommand loadMinimal(int commandStoreId, TxnId txnId, SavedCommand.Load load, RedundantBefore redundantBefore, DurableBefore durableBefore)
     {
         SavedCommand.Builder builder = loadDiffs(commandStoreId, txnId, load);
-        Cleanup cleanup = builder.shouldCleanup(redundantBefore, durableBefore);
+        if (!builder.nextCalled)
+            return null;
+
+        Cleanup cleanup = builder.shouldCleanup(node.agent(), redundantBefore, durableBefore);
         switch (cleanup)
         {
             case EXPUNGE_PARTIAL:
@@ -203,6 +209,7 @@ public class AccordJournal implements IJournal, Shutdownable
             case ERASE:
                 return null;
         }
+        Invariants.checkState(builder.saveStatus != null, "No saveSatus loaded, but next was called and cleanup was not: %s", builder);
         return builder.asMinimal();
     }
 
@@ -235,7 +242,7 @@ public class AccordJournal implements IJournal, Shutdownable
     }
 
     @Override
-    public void appendCommand(int store, SavedCommand.DiffWriter value, Runnable onFlush)
+    public void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush)
     {
         if (value == null || status == Status.REPLAY)
         {
@@ -248,7 +255,7 @@ public class AccordJournal implements IJournal, Shutdownable
         JournalKey key = new JournalKey(value.key(), JournalKey.Type.COMMAND_DIFF, store);
         RecordPointer pointer = journal.asyncWrite(key, value, SENTINEL_HOSTS);
         if (onFlush != null)
-            journal.onFlush(pointer, onFlush);
+            journal.onDurable(pointer, onFlush);
     }
 
     @Override
@@ -266,7 +273,7 @@ public class AccordJournal implements IJournal, Shutdownable
                 JournalKey key = new JournalKey(TxnId.NONE, JournalKey.Type.DURABLE_BEFORE, 0);
                 RecordPointer pointer = appendInternal(key, addDurableBefore);
                 // TODO (required): what happens on failure?
-                journal.onFlush(pointer, () -> result.setSuccess(null));
+                journal.onDurable(pointer, () -> result.setSuccess(null));
                 return result;
             }
 
@@ -297,7 +304,7 @@ public class AccordJournal implements IJournal, Shutdownable
             return;
 
         if (pointer != null)
-            journal.onFlush(pointer, onFlush);
+            journal.onDurable(pointer, onFlush);
         else
             onFlush.run();
     }
@@ -459,7 +466,7 @@ public class AccordJournal implements IJournal, Shutdownable
                         }
                     });
 
-                    Cleanup cleanup = builder.shouldCleanup(compactionInfo.redundantBefores.get(key.commandStoreId), compactionInfo.durableBefores.get(key.commandStoreId));
+                    Cleanup cleanup = builder.shouldCleanup(node.agent(), compactionInfo.redundantBefores.get(key.commandStoreId), compactionInfo.durableBefores.get(key.commandStoreId));
                     switch (cleanup)
                     {
                         case ERASE:
diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java b/src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java
index c6a2a46bf4..acda123245 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java
@@ -55,7 +55,7 @@ public class AccordJournalValueSerializers
     }
 
     public static class CommandDiffSerializer
-    implements FlyweightSerializer<SavedCommand.DiffWriter, SavedCommand.Builder>
+    implements FlyweightSerializer<SavedCommand.Writer, SavedCommand.Builder>
     {
         @Override
         public SavedCommand.Builder mergerFor(JournalKey journalKey)
@@ -64,7 +64,7 @@ public class AccordJournalValueSerializers
         }
 
         @Override
-        public void serialize(JournalKey key, SavedCommand.DiffWriter writer, DataOutputPlus out, int userVersion)
+        public void serialize(JournalKey key, SavedCommand.Writer writer, DataOutputPlus out, int userVersion)
         {
             try
             {
diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
index cf36e148e9..92fd117030 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
@@ -29,6 +29,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -57,8 +58,6 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.topology.Topology;
 import accord.utils.Invariants;
-import accord.utils.async.Observable;
-import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
@@ -111,12 +110,14 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.accord.RouteIndex;
+import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.LocalVersionedSerializer;
 import org.apache.cassandra.io.MessageVersionProvider;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
 import org.apache.cassandra.schema.KeyspaceMetadata;
@@ -149,6 +150,7 @@ import org.apache.cassandra.utils.concurrent.OpOrder;
 import static accord.utils.Invariants.checkArgument;
 import static accord.utils.Invariants.checkState;
 import static java.lang.String.format;
+import static java.util.Collections.emptyMap;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate;
 import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow;
@@ -183,7 +185,6 @@ public class AccordKeyspace
 
     private static final ClusteringIndexFilter FULL_PARTITION = new ClusteringIndexNamesFilter(BTreeSet.of(new ClusteringComparator(), Clustering.EMPTY), false);
 
-    //TODO (now, performance): should this be partitioner rather than TableId?  As of this patch distributed tables should only have 1 partitioner...
     private static final ConcurrentMap<TableId, AccordRoutingKeyByteSource.Serializer> TABLE_SERIALIZERS = new ConcurrentHashMap<>();
 
     private static AccordRoutingKeyByteSource.Serializer getRoutingKeySerializer(AccordRoutingKey key)
@@ -239,7 +240,10 @@ public class AccordKeyspace
               + "user_version int,"
               + "record blob,"
               + "PRIMARY KEY((store_id, type, id), descriptor, offset)"
-              + ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC) WITH compression = {'class':'NoopCompressor'};")
+              + ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC)" +
+              " WITH compression = {'class':'NoopCompressor'};")
+        .compaction(CompactionParams.lcs(emptyMap()))
+        .bloomFilterFpChance(0.01)
         .partitioner(new LocalPartitioner(BytesType.instance))
         .build();
 
@@ -478,7 +482,7 @@ public class AccordKeyspace
               + format("last_write_timestamp %s, ", TIMESTAMP_TUPLE)
               + "PRIMARY KEY((store_id, routing_key))"
               + ')')
-        .partitioner(new LocalPartitioner(CompositeType.getInstance(Int32Type.instance, BytesType.instance)))
+        .partitioner(new LocalCompositePrefixPartitioner(Int32Type.instance, BytesType.instance))
         .build();
 
     public static class TimestampsForKeyColumns
@@ -490,8 +494,14 @@ public class AccordKeyspace
         public static final ColumnMetadata last_executed_timestamp = getColumn(TimestampsForKeys, "last_executed_timestamp");
         public static final ColumnMetadata last_executed_micros = getColumn(TimestampsForKeys, "last_executed_micros");
         public static final ColumnMetadata last_write_timestamp = getColumn(TimestampsForKeys, "last_write_timestamp");
-
+        public static final ColumnMetadata last_write_id = getColumn(TimestampsForKeys, "last_write_id");
         static final Columns columns = Columns.from(Lists.newArrayList(last_executed_timestamp, last_executed_micros, last_write_timestamp));
+        static final ColumnFilter allColumns = ColumnFilter.all(TimestampsForKeys);
+
+        static DecoratedKey decorateKey(int storeId, RoutingKey key)
+        {
+            return TimestampsForKeys.partitioner.decorateKey(makeKey(storeId, key));
+        }
 
         static ByteBuffer makeKey(int storeId, RoutingKey key)
         {
@@ -517,6 +527,11 @@ public class AccordKeyspace
             return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
         }
 
+        public static TokenKey getKey(DecoratedKey key)
+        {
+            return getKey(splitPartitionKey(key));
+        }
+
         public static TokenKey getKey(ByteBuffer[] partitionKeyComponents)
         {
             return (TokenKey) deserializeRoutingKey(partitionKeyComponents[routing_key.position()]);
@@ -580,7 +595,7 @@ public class AccordKeyspace
         }
     }
 
-    private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance, BytesType.instance);
+    private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance);
     private static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
 
     private static TableMetadata commandsForKeysTable(String tableName)
@@ -596,6 +611,8 @@ public class AccordKeyspace
               + ')'
                + " WITH compression = {'class':'NoopCompressor'};")
         .partitioner(CFKPartitioner)
+        .compaction(CompactionParams.lcs(emptyMap()))
+        .bloomFilterFpChance(0.01)
         .build();
     }
 
@@ -983,7 +1000,7 @@ public class AccordKeyspace
     public static void findAllKeysBetween(int commandStore,
                                           AccordRoutingKey start, boolean startInclusive,
                                           AccordRoutingKey end, boolean endInclusive,
-                                          Observable<TokenKey> callback)
+                                          Consumer<TokenKey> consumer)
     {
 
         Token startToken = CommandsForKeysAccessor.getPrefixToken(commandStore, start);
@@ -1006,41 +1023,23 @@ public class AccordKeyspace
         else
             bounds = new ExcludingBounds<>(startPosition, endPosition);
 
-        Stage.READ.executor().submit(() -> {
-            ColumnFamilyStore baseCfs = Keyspace.openAndGetStore(CommandsForKeys);
-            try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
-                 WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
-                 CloseableIterator<DecoratedKey> iter = LocalCompositePrefixPartitioner.keyIterator(CommandsForKeys, bounds))
+        ColumnFamilyStore baseCfs = AccordColumnFamilyStores.commandsForKey;
+        try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
+             WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
+             CloseableIterator<DecoratedKey> iter = LocalCompositePrefixPartitioner.keyIterator(CommandsForKeys, bounds))
+        {
+            // Need the second try to handle callback errors vs read errors.
+            // Callback will see the read errors, but if the callback fails the outer try will see those errors
+            while (iter.hasNext())
             {
-                // Need the second try to handle callback errors vs read errors.
-                // Callback will see the read errors, but if the callback fails the outer try will see those errors
-                try
-                {
-                    while (iter.hasNext())
-                    {
-                        TokenKey pk = CommandsForKeysAccessor.getKey(iter.next());
-                        callback.onNext(pk);
-                    }
-                    callback.onCompleted();
-                }
-                catch (Exception e)
-                {
-                    callback.onError(e);
-                }
+                TokenKey pk = CommandsForKeysAccessor.getKey(iter.next());
+                consumer.accept(pk);
             }
-            catch (IOException e)
-            {
-                try
-                {
-                    callback.onError(e);
-                }
-                catch (Throwable t)
-                {
-                    e.addSuppressed(t);
-                }
-                throw new RuntimeException(e);
-            }
-        });
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
     }
 
     public static TxnId deserializeTxnId(UntypedResultSet.Row row)
@@ -1097,7 +1096,7 @@ public class AccordKeyspace
         return (TokenKey) AccordRoutingKeyByteSource.Serializer.fromComparableBytes(ByteBufferAccessor.instance, tokenBytes, tableId, currentVersion, null);
     }
 
-    public static Mutation getTimestampsForKeyMutation(int storeId, TimestampsForKey current, long timestampMicros)
+    public static PartitionUpdate getTimestampsForKeyUpdate(int storeId, TimestampsForKey current, long timestampMicros)
     {
         try
         {
@@ -1118,8 +1117,7 @@ public class AccordKeyspace
                 return null;
 
             ByteBuffer key = TimestampsForKeyColumns.makeKey(storeId, current.key());
-            PartitionUpdate update = singleRowUpdate(TimestampsForKeys, key, row);
-            return new Mutation(update);
+            return singleRowUpdate(TimestampsForKeys, key, row);
         }
         catch (IOException e)
         {
@@ -1127,50 +1125,79 @@ public class AccordKeyspace
         }
     }
 
-    public static Mutation getTimestampsForKeyMutation(AccordCommandStore commandStore, AccordSafeTimestampsForKey liveTimestamps, long timestampMicros)
+    public static Runnable getTimestampsForKeyUpdater(AccordCommandStore commandStore, TimestampsForKey liveTimestamps, long timestampMicros)
     {
-        return getTimestampsForKeyMutation(commandStore.id(), liveTimestamps.current(), timestampMicros);
+        PartitionUpdate upd = getTimestampsForKeyUpdate(commandStore.id(), liveTimestamps, timestampMicros);
+        return () -> {
+            ColumnFamilyStore cfs = AccordColumnFamilyStores.timestampsForKey;
+            try (OpOrder.Group group = Keyspace.writeOrder.start())
+            {
+                cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
+            }
+        };
     }
 
-    public static UntypedResultSet loadTimestampsForKeyRow(CommandStore commandStore, TokenKey key)
+    public static UntypedResultSet loadTimestampsForKeyRow(int commandStoreId, TokenKey key)
     {
         String cql = "SELECT * FROM " + ACCORD_KEYSPACE_NAME + '.' + TIMESTAMPS_FOR_KEY + ' ' +
                      "WHERE store_id = ? " +
                      "AND routing_key = ?";
 
-        return executeInternal(cql, commandStore.id(), serializeRoutingKey(key));
+        return executeInternal(cql, commandStoreId, serializeRoutingKey(key));
     }
 
-    public static TimestampsForKey loadTimestampsForKey(AccordCommandStore commandStore, TokenKey key)
+    private static SinglePartitionReadCommand getTimestampsForKeyRead(int storeId, TokenKey key, long nowInSeconds)
     {
-        commandStore.checkNotInStoreThread();
-        return unsafeLoadTimestampsForKey(commandStore, key);
+        return SinglePartitionReadCommand.create(TimestampsForKeys, nowInSeconds,
+                                                 TimestampsForKeyColumns.allColumns,
+                                                 RowFilter.none(),
+                                                 DataLimits.NONE,
+                                                 TimestampsForKeyColumns.decorateKey(storeId, key),
+                                                 FULL_PARTITION);
     }
 
-    public static TimestampsForKey unsafeLoadTimestampsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static TimestampsForKey loadTimestampsForKey(int commandStoreId, TokenKey key)
     {
-        UntypedResultSet rows = loadTimestampsForKeyRow(commandStore, key);
+        return unsafeLoadTimestampsForKey(commandStoreId, key);
+    }
 
-        if (rows.isEmpty())
+    public static TimestampsForKey unsafeLoadTimestampsForKey(int commandStoreId, TokenKey key)
+    {
+        long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
+        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
+
+        SinglePartitionReadCommand command = getTimestampsForKeyRead(commandStoreId, key, nowInSeconds);
+        try (ReadExecutionController controller = command.executionController();
+             FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
         {
-            return null;
+            if (!partitions.hasNext())
+                return null;
+
+            try (RowIterator partition = partitions.next())
+            {
+                Invariants.checkState(partition.hasNext());
+                Row row = partition.next();
+                TokenKey checkKey = TimestampsForKeyRows.getKey(partition.partitionKey());
+                checkState(checkKey.equals(key));
+
+                Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_executed_timestamp), ByteBufferAccessor.instance, Timestamp::fromBits, Timestamp.NONE);
+                ByteBuffer lastExecutedMicrosBB = cellValue(row, TimestampsForKeyColumns.last_executed_micros);
+                long lastExecutedMicros = lastExecutedMicrosBB == null || !lastExecutedMicrosBB.hasRemaining() ? 0 : lastExecutedMicrosBB.getLong(lastExecutedMicrosBB.position());
+                TxnId lastWriteId = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_write_id), ByteBufferAccessor.instance, TxnId::fromBits, TxnId.NONE);
+                Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(cellValue(row, TimestampsForKeyColumns.last_write_timestamp), ByteBufferAccessor.instance, Timestamp::fromBits, Timestamp.NONE);
+                return TimestampsForKey.SerializerSupport.create(key, lastExecutedTimestamp, lastExecutedMicros, lastWriteId, lastWriteTimestamp);
+            }
+        }
+        catch (Throwable t)
+        {
+            logger.error("Exception loading AccordTimestampsForKey " + key, t);
+            throw t;
         }
-
-        UntypedResultSet.Row row = rows.one();
-        TokenKey checkKey = (TokenKey) deserializeRoutingKey(row.getBytes("routing_key"));
-        checkState(checkKey.equals(key));
-
-        Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(row, "last_executed_timestamp", Timestamp::fromBits, Timestamp.NONE);
-        long lastExecutedMicros = row.has("last_executed_micros") ? row.getLong("last_executed_micros") : 0;
-        TxnId lastWriteId = deserializeTimestampOrDefault(row, "last_write_id", TxnId::fromBits, TxnId.NONE);
-        Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(row, "last_write_timestamp", Timestamp::fromBits, Timestamp.NONE);
-
-        return TimestampsForKey.SerializerSupport.create(key, lastExecutedTimestamp, lastExecutedMicros, lastWriteId, lastWriteTimestamp);
     }
 
-    private static DecoratedKey makeKeySeparateTable(CommandsForKeyAccessor accessor, int storeId, TokenKey key)
+    private static DecoratedKey makeKeySeparateTable(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
     {
-        ByteBuffer pk = accessor.keyComparator.make(storeId,
+        ByteBuffer pk = accessor.keyComparator.make(commandStoreId,
                                                     UUIDSerializer.instance.serialize(key.table().asUUID()),
                                                     serializeRoutingKeyNoTable(key)).serializeAsPartitionKey();
         return accessor.table.partitioner.decorateKey(pk);
@@ -1209,9 +1236,11 @@ public class AccordKeyspace
         return SchemaHolder.schema.getTablePartitioner(tableId);
     }
 
-    private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, long timestampMicros)
+    private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, Object serialized, long timestampMicros)
     {
-        ByteBuffer bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
+        ByteBuffer bytes;
+        if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized;
+        else bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
         return getCommandsForKeyPartitionUpdate(storeId, key, timestampMicros, bytes);
     }
 
@@ -1223,9 +1252,16 @@ public class AccordKeyspace
                                singleCellRow(Clustering.EMPTY, BufferCell.live(CommandsForKeysAccessor.data, timestampMicros, bytes)));
     }
 
-    public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKey update, long timestampMicros)
+    public static Runnable getCommandsForKeyUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros)
     {
-        return new Mutation(getCommandsForKeyPartitionUpdate(storeId, (TokenKey)update.key(), update, timestampMicros));
+        PartitionUpdate upd = getCommandsForKeyPartitionUpdate(storeId, key, update, serialized, timestampMicros);
+        return () -> {
+            ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey;
+            try (OpOrder.Group group = Keyspace.writeOrder.start())
+            {
+                cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
+            }
+        };
     }
 
     private static <T> ByteBuffer cellValue(Cell<T> cell)
@@ -1260,12 +1296,12 @@ public class AccordKeyspace
         return getCommandsForKeyRead(CommandsForKeysAccessor, storeId, key, nowInSeconds);
     }
 
-    static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, AccordCommandStore commandStore, TokenKey key)
+    static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
     {
         long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
         int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
 
-        SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStore.id(), key, nowInSeconds);
+        SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStoreId, key, nowInSeconds);
 
         try (ReadExecutionController controller = command.executionController();
              FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
@@ -1288,15 +1324,14 @@ public class AccordKeyspace
         }
     }
 
-    public static CommandsForKey unsafeLoadCommandsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static CommandsForKey unsafeLoadCommandsForKey(int commandStoreId, TokenKey key)
     {
-        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStoreId, key);
     }
 
-    public static CommandsForKey loadCommandsForKey(AccordCommandStore commandStore, TokenKey key)
+    public static CommandsForKey loadCommandsForKey(int commandStoreId, TokenKey key)
     {
-        commandStore.checkNotInStoreThread();
-        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStoreId, key);
     }
 
     public static class EpochDiskState
@@ -1626,4 +1661,11 @@ public class AccordKeyspace
         TABLE_SERIALIZERS.clear();
         SchemaHolder.schema = Schema.instance;
     }
+
+    public static class AccordColumnFamilyStores
+    {
+        public static final ColumnFamilyStore journal = Schema.instance.getColumnFamilyStoreInstance(Journal.id);
+        public static final ColumnFamilyStore commandsForKey = Schema.instance.getColumnFamilyStoreInstance(CommandsForKeys.id);
+        public static final ColumnFamilyStore timestampsForKey = Schema.instance.getColumnFamilyStoreInstance(TimestampsForKeys.id);
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
index e42b08b993..20f66f3b6a 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
@@ -272,7 +272,7 @@ public class AccordMessageSink implements MessageSink
         long delayedAtNanos = Long.MAX_VALUE;
         long expiresAtNanos;
         if (isRangeBarrier(request))
-            expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos();
+            expiresAtNanos = nowNanos + DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos();
         else
             expiresAtNanos = nowNanos + verb.expiresAfterNanos();
 
diff --git a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
index 3c55c06cf2..315649e304 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
@@ -379,6 +379,7 @@ public class AccordObjectSizes
         for (int i = 0 ; i < cfk.size() ; ++i)
         {
             TxnInfo info = cfk.get(i);
+            if (info.executeAt != info) size += TIMESTAMP_SIZE;
             if (info.getClass() != TxnInfoExtra.class) continue;
             TxnInfoExtra infoExtra = (TxnInfoExtra) info;
             if (infoExtra.missing.length > 0)
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java
index 0221e9f394..f2666fdfec 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java
@@ -19,25 +19,20 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.Objects;
-import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import accord.local.Command;
 import accord.local.SafeCommand;
 import accord.primitives.TxnId;
-import accord.utils.Invariants;
 import org.apache.cassandra.utils.concurrent.Ref;
 
-import static accord.utils.Invariants.Paranoia.LINEAR;
-import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
-
 public class AccordSafeCommand extends SafeCommand implements AccordSafeState<TxnId, Command>
 {
     public static class DebugAccordSafeCommand extends AccordSafeCommand
     {
         final Ref<?> selfRef;
-        public DebugAccordSafeCommand(AccordCachingState<TxnId, Command> global)
+        public DebugAccordSafeCommand(AccordCacheEntry<TxnId, Command> global)
         {
             super(global);
             selfRef = new Ref<>(this, null);
@@ -58,11 +53,11 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     }
 
     private boolean invalidated;
-    private final AccordCachingState<TxnId, Command> global;
+    private final AccordCacheEntry<TxnId, Command> global;
     private Command original;
     private Command current;
 
-    public AccordSafeCommand(AccordCachingState<TxnId, Command> global)
+    public AccordSafeCommand(AccordCacheEntry<TxnId, Command> global)
     {
         super(global.key());
         this.global = global;
@@ -97,7 +92,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     }
 
     @Override
-    public AccordCachingState<TxnId, Command> global()
+    public AccordCacheEntry<TxnId, Command> global()
     {
         checkNotInvalidated();
         return global;
@@ -125,7 +120,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
         return original;
     }
 
-    public SavedCommand.DiffWriter diff()
+    public SavedCommand.Writer diff()
     {
         return SavedCommand.diff(original, current);
     }
@@ -134,8 +129,10 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            uninitialised();
     }
 
     @Override
@@ -149,9 +146,4 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     {
         return invalidated;
     }
-
-    public static Function<AccordCachingState<TxnId, Command>, AccordSafeCommand> safeRefFactory()
-    {
-        return Invariants.testParanoia(LINEAR, LINEAR, HIGH) ? DebugAccordSafeCommand::new : AccordSafeCommand::new;
-    }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java
index eef12d6a7e..e28956cdc0 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java
@@ -18,6 +18,9 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
@@ -31,45 +34,48 @@ import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.api.RoutingKey;
-import accord.impl.AbstractSafeCommandStore;
 import accord.impl.CommandsSummary;
+import accord.impl.SafeTimestampsForKey;
 import accord.local.CommandStores;
 import accord.local.CommandStores.RangesForEpoch;
+import accord.local.KeyHistory;
 import accord.local.NodeCommandStoreService;
 import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
+import accord.local.SafeCommandStore;
 import accord.local.cfk.CommandsForKey;
 import accord.primitives.AbstractKeys;
 import accord.primitives.AbstractRanges;
 import accord.primitives.Ranges;
+import accord.primitives.Routable;
 import accord.primitives.Routables;
+import accord.primitives.RoutingKeys;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.utils.Invariants;
+import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
 
-public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey>
+import static accord.local.KeyHistory.TIMESTAMPS;
+import static accord.utils.Invariants.illegalArgument;
+import static accord.utils.Invariants.illegalState;
+
+public class AccordSafeCommandStore extends SafeCommandStore
 {
-    private final Map<TxnId, AccordSafeCommand> commands;
-    private final Map<RoutingKey, AccordSafeCommandsForKey> commandsForKeys;
-    private final Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKeys;
-    private final @Nullable AccordSafeCommandsForRanges commandsForRanges;
+    final AccordTask<?> task;
+    final PreLoadContext context;
+    private final @Nullable CommandsForRanges commandsForRanges;
     private final AccordCommandStore commandStore;
     private RangesForEpoch ranges;
     private FieldUpdates fieldUpdates;
 
-    private AccordSafeCommandStore(PreLoadContext context,
-                                   Map<TxnId, AccordSafeCommand> commands,
-                                   Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey,
-                                   Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey,
-                                   @Nullable AccordSafeCommandsForRanges commandsForRanges,
+    private AccordSafeCommandStore(AccordTask<?> task,
+                                   @Nullable CommandsForRanges commandsForRanges,
                                    AccordCommandStore commandStore)
     {
-        super(context);
-        this.commands = commands;
-        this.timestampsForKeys = timestampsForKey;
-        this.commandsForKeys = commandsForKey;
+        this.context = task.preLoadContext();
+        this.task = task;
         this.commandsForRanges = commandsForRanges;
         this.commandStore = commandStore;
         commandStore.updateRangesForEpoch(this);
@@ -77,80 +83,223 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
             this.ranges = Invariants.nonNull(commandStore.unsafeRangesForEpoch());
     }
 
-    public static AccordSafeCommandStore create(PreLoadContext preLoadContext,
-                                                Map<TxnId, AccordSafeCommand> commands,
-                                                Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey,
-                                                Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey,
-                                                @Nullable AccordSafeCommandsForRanges commandsForRanges,
+    public static AccordSafeCommandStore create(AccordTask<?> operation,
+                                                @Nullable CommandsForRanges commandsForRanges,
                                                 AccordCommandStore commandStore)
     {
-        return new AccordSafeCommandStore(preLoadContext, commands, timestampsForKey, commandsForKey, commandsForRanges, commandStore);
+        return new AccordSafeCommandStore(operation, commandsForRanges, commandStore);
+    }
+
+    @Override
+    public PreLoadContext canExecute(PreLoadContext context)
+    {
+        if (context.isEmpty()) return context;
+        if (context.keys().domain() == Routable.Domain.Range)
+            return context.isSubsetOf(this.context) ? context : null;
+
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return context.isSubsetOf(this.context) ? context : null;
+
+            for (TxnId txnId : context.txnIds())
+            {
+                if (null != getInternal(txnId))
+                    continue;
+
+                AccordSafeCommand safeCommand = caches.commands().acquireIfLoaded(txnId);
+                if (safeCommand == null)
+                    return null;
+
+                add(safeCommand, caches);
+            }
+
+            KeyHistory keyHistory = context.keyHistory();
+            if (keyHistory == KeyHistory.NONE)
+                return context;
+
+            List<RoutingKey> unavailable = null;
+            Unseekables<?> keys = context.keys();
+            if (keys.size() == 0)
+                return context;
+
+            for (int i = 0 ; i < keys.size() ; ++i)
+            {
+                RoutingKey key = (RoutingKey) keys.get(i);
+                if (keyHistory == TIMESTAMPS)
+                {
+                    if (null != timestampsForKeyInternal(key))
+                        continue; // already in working set
+
+                    AccordSafeTimestampsForKey safeTfk = caches.timestampsForKeys().acquireIfLoaded(key);
+                    if (safeTfk != null)
+                    {
+                        add(safeTfk, caches);
+                        continue;
+                    }
+                }
+                else
+                {
+                    if (null != getInternal(key))
+                        continue; // already in working set
+
+                    AccordSafeCommandsForKey safeCfk = caches.commandsForKeys().acquireIfLoaded(key);
+                    if (safeCfk != null)
+                    {
+                        add(safeCfk, caches);
+                        continue;
+                    }
+                }
+                if (unavailable == null)
+                    unavailable = new ArrayList<>();
+                unavailable.add(key);
+            }
+
+            if (unavailable == null)
+                return context;
+
+            if (unavailable.size() == keys.size())
+                return null;
+
+            return PreLoadContext.contextFor(context.primaryTxnId(), context.additionalTxnId(), keys.without(RoutingKeys.ofSortedUnique(unavailable)), keyHistory);
+        }
+    }
+
+    @Override
+    public PreLoadContext context()
+    {
+        return context;
     }
 
     @VisibleForTesting
     public Set<RoutingKey> commandsForKeysKeys()
     {
-        return commandsForKeys.keySet();
+        if (task.commandsForKey() == null)
+            return Collections.emptySet();
+        return task.commandsForKey().keySet();
     }
 
     @Override
-    protected AccordSafeCommand getCommandInternal(TxnId txnId)
+    protected AccordSafeCommand getInternal(TxnId txnId)
     {
+        Map<TxnId, AccordSafeCommand> commands = task.commands();
+        if (commands == null)
+            return null;
         return commands.get(txnId);
     }
 
     @Override
-    protected void addCommandInternal(AccordSafeCommand command)
+    protected AccordSafeCommand ifLoadedAndInitialisedAndNotErasedInternal(TxnId txnId)
     {
-        commands.put(command.txnId(), command);
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return null;
+
+            AccordSafeCommand command = caches.commands().acquireIfLoaded(txnId);
+            if (command == null)
+                return null;
+
+            return add(command, caches);
+        }
+    }
+
+    private AccordSafeCommand add(AccordSafeCommand safeCommand, ExclusiveCaches caches)
+    {
+        Object check = task.ensureCommands().putIfAbsent(safeCommand.txnId(), safeCommand);
+        if (check == null)
+        {
+            safeCommand.preExecute();
+            return safeCommand;
+        }
+        else
+        {
+            caches.commands().release(safeCommand, task);
+            throw illegalState("Attempted to take a duplicate reference to %s", safeCommand.txnId());
+        }
+    }
+
+    private AccordSafeCommandsForKey add(AccordSafeCommandsForKey safeCfk, ExclusiveCaches caches)
+    {
+        Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk);
+        if (check == null)
+        {
+            safeCfk.preExecute();
+            return safeCfk;
+        }
+        else
+        {
+            caches.commandsForKeys().release(safeCfk, task);
+            throw illegalState("Attempted to take a duplicate reference to CFK for %s", safeCfk.key());
+        }
+    }
+
+    private AccordSafeTimestampsForKey add(AccordSafeTimestampsForKey safeTfk, ExclusiveCaches caches)
+    {
+        Object check = task.ensureTimestampsForKey().putIfAbsent(safeTfk.key(), safeTfk);
+        if (check == null)
+        {
+            safeTfk.preExecute();
+            return safeTfk;
+        }
+        else
+        {
+            caches.timestampsForKeys().release(safeTfk, task);
+            throw illegalState("Attempted to take a duplicate reference to CFK for %s", safeTfk.key());
+        }
     }
 
     @Override
-    protected AccordSafeCommand getIfLoaded(TxnId txnId)
+    protected AccordSafeCommandsForKey getInternal(RoutingKey key)
     {
-        AccordSafeCommand command = commandStore.commandCache().acquireIfLoaded(txnId);
-        if (command != null) command.preExecute();
-        return command;
+        Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey = task.commandsForKey();
+        if (commandsForKey == null)
+            return null;
+        return commandsForKey.get(key);
     }
 
     @Override
-    protected AccordSafeCommandsForKey getCommandsForKeyInternal(RoutingKey key)
+    protected AccordSafeCommandsForKey ifLoadedInternal(RoutingKey key)
     {
-        return commandsForKeys.get(key);
+        try (ExclusiveCaches caches = commandStore.tryLockCaches())
+        {
+            if (caches == null)
+                return null;
+
+            AccordSafeCommandsForKey safeCfk = caches.commandsForKeys().acquireIfLoaded(key);
+            if (safeCfk == null)
+                return null;
+
+            Object check = task.ensureCommandsForKey().putIfAbsent(safeCfk.key(), safeCfk);
+            if (check == null)
+            {
+                safeCfk.preExecute();
+                return safeCfk;
+            }
+            else
+            {
+                caches.commandsForKeys().release(safeCfk, task);
+                throw illegalState("Attempted to take a duplicate reference to CFK for %s", key);
+            }
+        }
     }
 
     @Override
-    protected void addCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
+    public SafeTimestampsForKey timestampsForKey(RoutingKey key)
     {
-        commandsForKeys.put(cfk.key(), cfk);
+        AccordSafeTimestampsForKey safeTfk = timestampsForKeyInternal(key);
+        if (safeTfk == null)
+            throw illegalArgument("%s not referenced in %s", key, context);
+        return safeTfk;
     }
 
-    @Override
-    protected AccordSafeCommandsForKey getCommandsForKeyIfLoaded(RoutingKey key)
+    private AccordSafeTimestampsForKey timestampsForKeyInternal(RoutingKey key)
     {
-        AccordSafeCommandsForKey cfk = commandStore.commandsForKeyCache().acquireIfLoaded(key);
-        if (cfk != null) cfk.preExecute();
-        return cfk;
-    }
+        Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey = task.timestampsForKey();
+        if (timestampsForKey == null)
+            return null;
 
-    @Override
-    protected AccordSafeTimestampsForKey getTimestampsForKeyInternal(RoutingKey key)
-    {
-        return timestampsForKeys.get(key);
-    }
-
-    @Override
-    protected void addTimestampsForKeyInternal(AccordSafeTimestampsForKey cfk)
-    {
-        timestampsForKeys.put(cfk.key(), cfk);
-    }
-
-    @Override
-    protected AccordSafeTimestampsForKey getTimestampsForKeyIfLoaded(RoutingKey key)
-    {
-        AccordSafeTimestampsForKey cfk = commandStore.timestampsForKeyCache().acquireIfLoaded(key);
-        if (cfk != null) cfk.preExecute();
-        return cfk;
+        return timestampsForKey.get(key);
     }
 
     @Override
@@ -200,27 +349,27 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
     {
         if (commandsForRanges == null)
             return accumulate;
-        CommandsForRanges cfr = commandsForRanges.current();
+
         switch (keysOrRanges.domain())
         {
             case Key:
             {
                 AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
-                if (!cfr.ranges.intersects(keys))
+                if (!commandsForRanges.ranges.intersects(keys))
                     return accumulate;
             }
             break;
             case Range:
             {
                 AbstractRanges ranges = (AbstractRanges) keysOrRanges;
-                if (!cfr.ranges.intersects(ranges))
+                if (!commandsForRanges.ranges.intersects(ranges))
                     return accumulate;
             }
             break;
             default:
                 throw new AssertionError("Unknown domain: " + keysOrRanges.domain());
         }
-        return map.apply(cfr, accumulate);
+        return map.apply(commandsForRanges, accumulate);
     }
 
     private <O> O mapReduceForKey(Routables<?> keysOrRanges, BiFunction<CommandsSummary, O, O> map, O accumulate)
@@ -246,7 +395,12 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
                 // are contained within the ranges... so walk all keys found in commandsForKeys
                 if (!context.keys().containsAll(keysOrRanges))
                     throw new AssertionError("Range(s) detected not present in the PreLoadContext: expected " + context.keys() + " but given " + keysOrRanges);
-                for (RoutingKey key : commandsForKeys.keySet())
+
+                Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey = task.commandsForKey();
+                if (commandsForKey == null)
+                    break;
+
+                for (RoutingKey key : commandsForKey.keySet())
                 {
                     //TODO (duplicate code): this is a repeat of Key... only change is checking contains in range
                     CommandsForKey commands = get(key).current();
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java
index 5874442528..634abda68b 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java
@@ -29,11 +29,11 @@ import accord.local.cfk.SafeCommandsForKey;
 public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState<RoutingKey, CommandsForKey>
 {
     private boolean invalidated;
-    private final AccordCachingState<RoutingKey, CommandsForKey> global;
+    private final AccordCacheEntry<RoutingKey, CommandsForKey> global;
     private CommandsForKey original;
     private CommandsForKey current;
 
-    public AccordSafeCommandsForKey(AccordCachingState<RoutingKey, CommandsForKey> global)
+    public AccordSafeCommandsForKey(AccordCacheEntry<RoutingKey, CommandsForKey> global)
     {
         super(global.key());
         this.global = global;
@@ -82,7 +82,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     }
 
     @Override
-    public AccordCachingState<RoutingKey, CommandsForKey> global()
+    public AccordCacheEntry<RoutingKey, CommandsForKey> global()
     {
         checkNotInvalidated();
         return global;
@@ -113,8 +113,10 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            initialize();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForRanges.java b/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForRanges.java
deleted file mode 100644
index 1a90c0a700..0000000000
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForRanges.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.util.NavigableMap;
-import java.util.Objects;
-
-import accord.primitives.Ranges;
-import accord.primitives.TxnId;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import org.apache.cassandra.utils.Pair;
-
-public class AccordSafeCommandsForRanges extends ImmutableAccordSafeState<Ranges, CommandsForRanges>
-{
-    private final AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain;
-
-    public AccordSafeCommandsForRanges(Ranges ranges, AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain)
-    {
-        super(ranges);
-        this.chain = chain;
-    }
-
-    public Ranges ranges()
-    {
-        return key();
-    }
-
-    @Override
-    public void preExecute()
-    {
-        checkNotInvalidated();
-        Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>> pair = AsyncChains.getUnchecked(chain);
-        pair.left.close();
-        pair.left.get().entrySet().forEach(e -> pair.right.put(e.getKey(), e.getValue()));
-        original = CommandsForRanges.create(key, pair.right);
-    }
-
-    @Override
-    public AccordCachingState<Ranges, CommandsForRanges> global()
-    {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        AccordSafeCommandsForRanges that = (AccordSafeCommandsForRanges) o;
-        return Objects.equals(original, that.original);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(original);
-    }
-
-    @Override
-    public String toString()
-    {
-        return "AccordSafeCommandsForRange{" +
-               "chain=" + chain +
-               ", invalidated=" + invalidated +
-               ", original=" + original +
-               '}';
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java
index d8e950d063..66b3eb1d1d 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeState.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSafeState.java
@@ -18,8 +18,7 @@
 package org.apache.cassandra.service.accord;
 
 import accord.impl.SafeState;
-import accord.utils.async.AsyncChain;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
+import accord.utils.async.Cancellable;
 
 public interface AccordSafeState<K, V> extends SafeState<V>
 {
@@ -29,7 +28,7 @@ public interface AccordSafeState<K, V> extends SafeState<V>
     boolean invalidated();
     void preExecute();
 
-    AccordCachingState<K, V> global();
+    AccordCacheEntry<K, V> global();
 
     default boolean hasUpdate()
     {
@@ -46,17 +45,7 @@ public interface AccordSafeState<K, V> extends SafeState<V>
         return global().key();
     }
 
-    default Status globalStatus()
-    {
-        return global().status();
-    }
-
-    default AsyncChain<?> loading()
-    {
-        return global().loading();
-    }
-
-    default AsyncChain<?> saving()
+    default Cancellable saving()
     {
         return global().saving();
     }
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java b/src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java
index 77ad56c3fd..6406f53704 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java
@@ -31,11 +31,11 @@ import accord.primitives.Timestamp;
 public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements AccordSafeState<RoutingKey, TimestampsForKey>
 {
     private boolean invalidated;
-    private final AccordCachingState<RoutingKey, TimestampsForKey> global;
+    private final AccordCacheEntry<RoutingKey, TimestampsForKey> global;
     private TimestampsForKey original;
     private TimestampsForKey current;
 
-    public AccordSafeTimestampsForKey(AccordCachingState<RoutingKey, TimestampsForKey> global)
+    public AccordSafeTimestampsForKey(AccordCacheEntry<RoutingKey, TimestampsForKey> global)
     {
         super(global.key());
         this.global = global;
@@ -70,7 +70,7 @@ public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements
     }
 
     @Override
-    public AccordCachingState<RoutingKey, TimestampsForKey> global()
+    public AccordCacheEntry<RoutingKey, TimestampsForKey> global()
     {
         checkNotInvalidated();
         return global;
@@ -101,8 +101,10 @@ public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements
     public void preExecute()
     {
         checkNotInvalidated();
-        original = global.get();
+        original = global.getExclusive();
         current = original;
+        if (isUnset())
+            initialize();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java b/src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
index db72c9f932..719ade8548 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
@@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.PartitionUpdate.SimpleBuilder;
@@ -73,7 +72,7 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
         if (readers.isEmpty())
             return Collections.emptyList();
 
-        ColumnFamilyStore cfs = Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL);
+        ColumnFamilyStore cfs = AccordKeyspace.AccordColumnFamilyStores.journal;
         Descriptor descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables());
         SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
 
diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java
index 5433b09145..830855d5ff 100644
--- a/src/java/org/apache/cassandra/service/accord/AccordService.java
+++ b/src/java/org/apache/cassandra/service/accord/AccordService.java
@@ -82,8 +82,10 @@ import accord.local.Node;
 import accord.local.Node.Id;
 import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
+import accord.local.SafeCommand;
 import accord.local.ShardDistributor.EvenSplit;
 import accord.local.cfk.CommandsForKey;
+import accord.local.cfk.SafeCommandsForKey;
 import accord.messages.Callback;
 import accord.messages.ReadData;
 import accord.messages.Reply;
@@ -127,9 +129,9 @@ import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.AccordClientRequestMetrics;
-import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.metrics.ClientRequestMetrics;
 import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessageDelivery;
@@ -181,6 +183,7 @@ import static accord.utils.Invariants.checkState;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordCommandStoreShardCount;
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
@@ -269,6 +272,9 @@ public class AccordService implements IAccordService, Shutdownable
         @Override
         public void setCacheSize(long kb) { }
 
+        @Override
+        public void setWorkingSetSize(long kb) { }
+
         @Override
         public TopologyManager topology()
         {
@@ -312,6 +318,12 @@ public class AccordService implements IAccordService, Shutdownable
             return new CompactionInfo(new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>());
         }
 
+        @Override
+        public AccordAgent agent()
+        {
+            return null;
+        }
+
         @Override
         public List<CommandStoreTxnBlockedGraph> debugTxnBlockedGraph(TxnId txnId)
         {
@@ -427,13 +439,13 @@ public class AccordService implements IAccordService, Shutdownable
         this.scheduler = new AccordScheduler();
         this.dataStore = new AccordDataStore();
         this.configuration = new AccordConfiguration(DatabaseDescriptor.getRawConfig());
-        this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal);
+        this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal, agent);
         this.node = new Node(localId,
                              messageSink,
                              configService,
                              time,
                              () -> dataStore,
-                             new KeyspaceSplitter(new EvenSplit<>(DatabaseDescriptor.getAccordShardCount(), getPartitioner().accordSplitter())),
+                             new KeyspaceSplitter(new EvenSplit<>(getAccordCommandStoreShardCount(), getPartitioner().accordSplitter())),
                              agent,
                              new DefaultRandom(),
                              scheduler,
@@ -624,13 +636,13 @@ public class AccordService implements IAccordService, Shutdownable
             AsyncResult<TxnId> asyncResult = syncPoint == null
                                                  ? Barrier.barrier(node, keysOrRanges, route, epoch, barrierType)
                                                  : Barrier.barrier(node, keysOrRanges, route, epoch, barrierType, syncPoint);
+            long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos;
+            TxnId txnId = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
             if (keysOrRanges.domain() == Key)
             {
                 PartitionKey key = (PartitionKey)keysOrRanges.get(0);
-                asyncResult.accept(txnId -> maybeSaveAccordKeyMigrationLocally(key, Epoch.create(txnId.epoch())));
+                maybeSaveAccordKeyMigrationLocally(key, Epoch.create(txnId.epoch()));
             }
-            long deadlineNanos = requestTime.startedAtNanos() + timeoutNanos;
-            TxnId txnId = AsyncChains.getBlocking(asyncResult, deadlineNanos - nanoTime(), NANOSECONDS);
             ((AccordAgent) node.agent()).onSuccessfulBarrier(txnId, keysOrRanges);
             logger.debug("Completed barrier attempt in {}ms, {}ms since attempts start, barrier key: {} epoch: {} barrierType: {} isForWrite {}",
                          sw.elapsed(MILLISECONDS),
@@ -838,7 +850,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Override
     public Seekables barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
     {
-        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite),
+        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().barrier(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite),
                              DatabaseDescriptor.getAccordBarrierRetryAttempts(),
                              DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(),
                              DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis());
@@ -847,7 +859,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Override
     public Seekables<?, ?> repairWithRetries(Seekables<?, ?> keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite, List<InetAddressAndPort> allEndpoints) throws InterruptedException
     {
-        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeBarrierTimeoutNanos(), barrierType, isForWrite, allEndpoints),
+        return doWithRetries(Blocking.Default.instance, () -> AccordService.instance().repair(keysOrRanges, minEpoch, Dispatcher.RequestTime.forImmediateExecution(), DatabaseDescriptor.getAccordRangeSyncPointTimeoutNanos(), barrierType, isForWrite, allEndpoints),
                              DatabaseDescriptor.getAccordBarrierRetryAttempts(),
                              DatabaseDescriptor.getAccordBarrierRetryInitialBackoffMillis(),
                              DatabaseDescriptor.getAccordBarrierRetryMaxBackoffMillis());
@@ -1018,6 +1030,14 @@ public class AccordService implements IAccordService, Shutdownable
         commandStores.setCapacity(bytes);
     }
 
+    @Override
+    public void setWorkingSetSize(long kb)
+    {
+        long bytes = kb << 10;
+        AccordCommandStores commandStores = (AccordCommandStores) node.commandStores();
+        commandStores.setWorkingSetSize(bytes);
+    }
+
     @Override
     public boolean isTerminated()
     {
@@ -1126,7 +1146,7 @@ public class AccordService implements IAccordService, Shutdownable
 
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, CommandStore commandStore, TokenKey blockedBy, TxnId txnId, Timestamp executeAt)
     {
-        AsyncChain<AsyncChain<Void>> submit = commandStore.submit(PreLoadContext.contextFor(txnId, RoutingKeys.of(blockedBy.toUnseekable()), KeyHistory.COMMANDS), in -> {
+        AsyncChain<AsyncChain<Void>> submit = commandStore.submit(PreLoadContext.contextFor(txnId, RoutingKeys.of(blockedBy.toUnseekable()), KeyHistory.SYNC), in -> {
             AsyncChain<Void> chain = populate(state, (AccordSafeCommandStore) in, blockedBy, txnId, executeAt);
             return chain == null ? AsyncChains.success(null) : chain;
         });
@@ -1136,7 +1156,7 @@ public class AccordService implements IAccordService, Shutdownable
     @Nullable
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, AccordSafeCommandStore safeStore, TxnId txnId)
     {
-        AccordSafeCommand safeCommand = safeStore.getIfLoaded(txnId);
+        SafeCommand safeCommand = safeStore.unsafeGet(txnId);
         Invariants.nonNull(safeCommand, "Txn %s is not in the cache", txnId);
         if (safeCommand.current() == null || safeCommand.current().saveStatus() == SaveStatus.Uninitialised)
             return null;
@@ -1149,7 +1169,7 @@ public class AccordService implements IAccordService, Shutdownable
         {
             if (state.knows(blockedBy)) continue;
             // need to fetch the state
-            if (safeStore.getIfLoaded(blockedBy) != null)
+            if (safeStore.ifLoadedAndInitialisedAndNotErased(blockedBy) != null)
             {
                 AsyncChain<Void> chain = populate(state, safeStore, blockedBy);
                 if (chain != null)
@@ -1164,7 +1184,7 @@ public class AccordService implements IAccordService, Shutdownable
         for (TokenKey blockedBy : cmdTxnState.blockedByKey)
         {
             if (state.keys.containsKey(blockedBy)) continue;
-            if (safeStore.getCommandsForKeyIfLoaded(blockedBy) != null)
+            if (safeStore.ifLoadedAndInitialised(blockedBy) != null)
             {
                 AsyncChain<Void> chain = populate(state, safeStore, blockedBy, txnId, safeCommand.current().executeAt());
                 if (chain != null)
@@ -1183,13 +1203,13 @@ public class AccordService implements IAccordService, Shutdownable
 
     private static AsyncChain<Void> populate(CommandStoreTxnBlockedGraph.Builder state, AccordSafeCommandStore safeStore, TokenKey pk, TxnId txnId, Timestamp executeAt)
     {
-        AccordSafeCommandsForKey commandsForKey = safeStore.getCommandsForKeyIfLoaded(pk);
+        SafeCommandsForKey commandsForKey = safeStore.ifLoadedAndInitialised(pk);
         TxnId blocking = commandsForKey.current().blockedOnTxnId(txnId, executeAt);
         if (blocking instanceof CommandsForKey.TxnInfo)
             blocking = ((CommandsForKey.TxnInfo) blocking).plainTxnId();
         state.keys.put(pk, blocking);
         if (state.txns.containsKey(blocking)) return null;
-        if (safeStore.getIfLoaded(blocking) != null) return populate(state, safeStore, blocking);
+        if (safeStore.ifLoadedAndInitialisedAndNotErased(blocking) != null) return populate(state, safeStore, blocking);
         return populate(state, safeStore.commandStore(), blocking);
     }
 
@@ -1354,6 +1374,12 @@ public class AccordService implements IAccordService, Shutdownable
         return new CompactionInfo(redundantBefores, ranges, durableBefores);
     }
 
+    @Override
+    public AccordAgent agent()
+    {
+        return (AccordAgent) node.agent();
+    }
+
     @Override
     public void awaitTableDrop(TableId id)
     {
diff --git a/src/java/org/apache/cassandra/service/accord/AccordStateCache.java b/src/java/org/apache/cassandra/service/accord/AccordStateCache.java
deleted file mode 100644
index 4c3f068976..0000000000
--- a/src/java/org/apache/cassandra/service/accord/AccordStateCache.java
+++ /dev/null
@@ -1,787 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.ToLongFunction;
-import java.util.stream.Stream;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.utils.IntrusiveLinkedList;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChains;
-import org.agrona.collections.Int2ObjectHashMap;
-import org.apache.cassandra.cache.CacheSize;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
-import org.apache.cassandra.metrics.CacheAccessMetrics;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
-
-import static accord.utils.Invariants.checkState;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADING;
-import static org.apache.cassandra.service.accord.AccordCachingState.Status.SAVING;
-
-/**
- * Cache for AccordCommand and AccordCommandsForKey, available memory is shared between the two object types.
- * </p>
- * Supports dynamic object sizes. After each acquire/free cycle, the cacheable objects size is recomputed to
- * account for data added/removed during txn processing if it's modified flag is set
- */
-public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?>> implements CacheSize
-{
-    private static final Logger logger = LoggerFactory.getLogger(AccordStateCache.class);
-
-    // Debug mode to verify that loading from journal + system tables results in
-    // functionally identical (or superceding) command to the one we've just evicted.
-    private static boolean VALIDATE_LOAD_ON_EVICT = false;
-
-    @VisibleForTesting
-    public static void validateLoadOnEvict(boolean value)
-    {
-        VALIDATE_LOAD_ON_EVICT = value;
-    }
-
-    static class Stats
-    {
-        long queries;
-        long hits;
-        long misses;
-    }
-
-    public static final class ImmutableStats
-    {
-        public final long queries;
-        public final long hits;
-        public final long misses;
-        
-        public ImmutableStats(Stats stats)
-        {
-            queries = stats.queries;
-            hits = stats.hits;
-            misses = stats.misses;
-        }
-    }
-
-    // TODO (required): cleanup on drop table, or else share between command stores
-    private Int2ObjectHashMap<Instance<?, ?, ?>> instances = new Int2ObjectHashMap<>();
-    private int nextIndex;
-
-    private final ExecutorPlus loadExecutor, saveExecutor;
-
-    private int unreferenced = 0;
-    private long maxSizeInBytes;
-    private long bytesCached = 0;
-
-    @VisibleForTesting
-    final AccordStateCacheMetrics metrics;
-    final Stats stats = new Stats();
-
-    public AccordStateCache(ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, long maxSizeInBytes, AccordStateCacheMetrics metrics)
-    {
-        this.loadExecutor = loadExecutor;
-        this.saveExecutor = saveExecutor;
-        this.maxSizeInBytes = maxSizeInBytes;
-        this.metrics = metrics;
-    }
-
-    @Override
-    public void setCapacity(long sizeInBytes)
-    {
-        maxSizeInBytes = sizeInBytes;
-        maybeEvictSomeNodes();
-    }
-
-    @Override
-    public long capacity()
-    {
-        return maxSizeInBytes;
-    }
-
-    private void unlink(AccordCachingState<?, ?> node)
-    {
-        node.unlink();
-        unreferenced--;
-    }
-
-    private void link(AccordCachingState<?, ?> node)
-    {
-        addLast(node);
-        unreferenced++;
-    }
-
-    @SuppressWarnings("unchecked")
-    private <K, V> void maybeUpdateSize(AccordCachingState<?, ?> node, ToLongFunction<?> estimator)
-    {
-        if (node.shouldUpdateSize())
-        {
-            long delta = ((AccordCachingState<K, V>) node).estimatedSizeOnHeapDelta((ToLongFunction<V>) estimator);
-            bytesCached += delta;
-            instanceForNode(node).bytesCached += delta;
-        }
-    }
-
-    /*
-     * Roughly respects LRU semantics when evicting. Might consider prioritising keeping MODIFIED nodes around
-     * for longer to maximise the chances of hitting system tables fewer times (or not at all).
-     */
-    private void maybeEvictSomeNodes()
-    {
-        if (bytesCached <= maxSizeInBytes)
-            return;
-
-        Iterator<AccordCachingState<?, ?>> iter = this.iterator();
-        while (iter.hasNext() && bytesCached > maxSizeInBytes)
-        {
-            AccordCachingState<?, ?> node = iter.next();
-            maybeEvict(node);
-        }
-    }
-
-    @VisibleForTesting
-    public boolean maybeEvict(AccordCachingState<?, ?> node)
-    {
-        checkState(node.references == 0);
-
-        /*
-         * TODO (expected, efficiency):
-         *    can this be reworked so we're not skipping unevictable nodes everytime we try to evict?
-         */
-        Status status = node.status(); // status() call completes (if completeable)
-        switch (status)
-        {
-            default: throw new IllegalStateException("Unhandled status " + status);
-            case LOADED:
-                unlink(node);
-                evict(node);
-                return true;
-            case MODIFIED:
-                // schedule a save to disk, keep linked and in the cache map
-                Instance<?, ?, ?> instance = instanceForNode(node);
-                node.save(saveExecutor, instance.saveFunction);
-                maybeUpdateSize(node, instance.heapEstimator);
-                return false;
-            case SAVING:
-                // skip over until completes to LOADED or FAILED_TO_SAVE
-                return false;
-            case FAILED_TO_SAVE:
-                // TODO (consider): panic when a save fails
-                // permanently unlink, but keep in the map
-                unlink(node);
-                return false;
-        }
-    }
-
-    private boolean isInQueue(AccordCachingState<?, ?> node)
-    {
-        return node.isLinked();
-    }
-
-    private void evict(AccordCachingState<?, ?> node)
-    {
-        if (logger.isTraceEnabled())
-            logger.trace("Evicting {} {} - {}", node.status(), node.key(), node.isLoaded() ? node.get() : null);
-
-        checkState(!isInQueue(node));
-
-        bytesCached -= node.lastQueriedEstimatedSizeOnHeap;
-        Instance<?, ?, ?> instance = instanceForNode(node);
-        instance.bytesCached -= node.lastQueriedEstimatedSizeOnHeap;
-
-        if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT)
-            instance.validateLoadEvicted(node);
-
-        AccordCachingState<?, ?> self = instances.get(node.index).cache.remove(node.key());
-        Invariants.checkState(self.references == 0);
-        checkState(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self);
-        if (instance.listeners != null)
-            instance.listeners.forEach(l -> l.onEvict((AccordCachingState) node));
-    }
-
-    public ImmutableStats stats()
-    {
-        return new ImmutableStats(stats);
-    }
-
-    private Instance<?, ?, ?> instanceForNode(AccordCachingState<?, ?> node)
-    {
-        return instances.get(node.index);
-    }
-
-    public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
-        Class<K> keyClass,
-        Class<S> valClass,
-        Function<AccordCachingState<K, V>, S> safeRefFactory,
-        Function<K, V> loadFunction,
-        Function<V, Runnable> saveFunction,
-        BiFunction<K, V, Boolean> validateFunction,
-        ToLongFunction<V> heapEstimator,
-        AccordCachingState.Factory<K, V> nodeFactory)
-    {
-        int index = ++nextIndex;
-
-        Instance<K, V, S> instance =
-            new Instance<>(index, keyClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, nodeFactory);
-
-        Int2ObjectHashMap<Instance<?, ?, ?>> newInstances = new Int2ObjectHashMap<>(instances);
-        newInstances.put(index, instance);
-        instances = newInstances;
-
-        return instance;
-    }
-
-    public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
-        Class<K> keyClass,
-        Class<S> valClass,
-        Function<AccordCachingState<K, V>, S> safeRefFactory,
-        Function<K, V> loadFunction,
-        Function<V, Runnable> saveFunction,
-        BiFunction<K, V, Boolean> validateFunction,
-        ToLongFunction<V> heapEstimator)
-    {
-        return instance(keyClass, valClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, AccordCachingState.defaultFactory());
-    }
-
-    public Collection<Instance<?, ? ,? >> instances()
-    {
-        return instances.values();
-    }
-
-    public interface Listener<K, V>
-    {
-        default void onAdd(AccordCachingState<K, V> state) {}
-        default void onRelease(AccordCachingState<K, V> state) {}
-        default void onEvict(AccordCachingState<K, V> state) {}
-    }
-
-    public class Instance<K, V, S extends AccordSafeState<K, V>> implements CacheSize, Iterable<AccordCachingState<K, V>>
-    {
-        private final int index;
-        private final Class<K> keyClass;
-        private final Function<AccordCachingState<K, V>, S> safeRefFactory;
-        private Function<K, V> loadFunction;
-        private Function<V, Runnable> saveFunction;
-        private final BiFunction<K, V, Boolean> validateFunction;
-        private final ToLongFunction<V> heapEstimator;
-        private long bytesCached;
-
-        @VisibleForTesting
-        final CacheAccessMetrics instanceMetrics;
-        private final Stats stats = new Stats();
-        private final Map<Object, AccordCachingState<?, ?>> cache = new HashMap<>();
-        private final AccordCachingState.Factory<K, V> nodeFactory;
-        private List<Listener<K, V>> listeners = null;
-
-        public Instance(
-            int index, Class<K> keyClass,
-            Function<AccordCachingState<K, V>, S> safeRefFactory,
-            Function<K, V> loadFunction,
-            Function<V, Runnable> saveFunction,
-            BiFunction<K, V, Boolean> validateFunction,
-            ToLongFunction<V> heapEstimator,
-            AccordCachingState.Factory<K, V> nodeFactory)
-        {
-            this.index = index;
-            this.keyClass = keyClass;
-            this.safeRefFactory = safeRefFactory;
-            this.loadFunction = loadFunction;
-            this.saveFunction = saveFunction;
-            this.validateFunction = validateFunction;
-            this.heapEstimator = heapEstimator;
-            this.instanceMetrics = metrics.forInstance(keyClass);
-            this.nodeFactory = nodeFactory;
-        }
-
-        public void register(Listener<K, V> l)
-        {
-            if (listeners == null)
-                listeners = new ArrayList<>();
-            listeners.add(l);
-        }
-
-        public void unregister(Listener<K, V> l)
-        {
-            if (listeners == null)
-                throw new AssertionError("No listeners exist");
-            if (!listeners.remove(l))
-                throw new AssertionError("Listener was not registered");
-            if (listeners.isEmpty())
-                listeners = null;
-        }
-
-        public Stream<AccordCachingState<K, V>> stream()
-        {
-            return cache.entrySet().stream()
-                        .filter(e -> instanceForNode(e.getValue()) == this)
-                        .map(e -> (AccordCachingState<K, V>) e.getValue());
-        }
-
-        @Override
-        public Iterator<AccordCachingState<K, V>> iterator()
-        {
-            return stream().iterator();
-        }
-
-        public S acquireOrInitialize(K key, Function<K, V> valueFactory)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-            {
-                node = nodeFactory.create(key, index);
-                node.initialize(valueFactory.apply(key));
-                cache.put(key, node);
-                if (listeners != null)
-                {
-                    AccordCachingState<K, V> finalNode = node;
-                    listeners.forEach(l -> l.onAdd(finalNode));
-                }
-            }
-            AccordCachingState<K, V> acquired = acquireExisting(node, true, null);
-            Invariants.checkState(acquired != null, "%s could not be acquired", node);
-            return safeRefFactory.apply(acquired);
-        }
-
-        public S acquireIfExists(K key)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-                return null;
-
-            return safeRefFactory.apply(acquireExisting(node, false, null));
-        }
-
-        public void maybeLoad(K key, V initial)
-        {
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            if (node == null)
-            {
-                node = nodeFactory.create(key, index);
-                node.initialize(initial);
-                Object prev = cache.put(key, node);
-                Invariants.checkState(prev == null, "%s not absent from cache: %s already present", key, node);
-                if (listeners != null)
-                {
-                    AccordCachingState<K, V> finalNode = node;
-                    listeners.forEach(l -> l.onAdd(finalNode));
-                }
-                maybeUpdateSize(node, heapEstimator);
-            }
-        }
-
-        public S acquire(K key)
-        {
-            return acquire(key, null);
-        }
-
-        public S acquireIfLoaded(K key)
-        {
-            return acquireIfLoaded(key, null);
-        }
-
-        public S acquire(K key, @Nullable ExecutorPlus loadExecutor)
-        {
-            AccordCachingState<K, V> node = acquire(key, false, loadExecutor);
-            return safeRefFactory.apply(node);
-        }
-
-        public S acquireIfLoaded(K key, @Nullable ExecutorPlus loadExecutor)
-        {
-            AccordCachingState<K, V> node = acquire(key, true, loadExecutor);
-            if (node == null)
-                return null;
-            return safeRefFactory.apply(node);
-        }
-
-        private AccordCachingState<K, V> acquire(K key, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            incrementCacheQueries();
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node == null
-                 ? acquireAbsent(key, onlyIfLoaded, loadExecutor)
-                 : acquireExisting(node, onlyIfLoaded, loadExecutor);
-        }
-
-        /*
-         * Can only return a LOADING Node (or null)
-         */
-        private AccordCachingState<K, V> acquireAbsent(K key, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            incrementCacheMisses();
-            if (onlyIfLoaded)
-                return null;
-            AccordCachingState<K, V> node = nodeFactory.create(key, index);
-            if (loadExecutor == null)
-                loadExecutor = AccordStateCache.this.loadExecutor;
-            node.load(loadExecutor, loadFunction);
-            node.references++;
-
-            Object prev = cache.put(key, node);
-            Invariants.checkState(prev == null, "%s not absent from cache: %s already present", key, node);
-            if (listeners != null)
-                listeners.forEach(l -> l.onAdd(node));
-            maybeUpdateSize(node, heapEstimator);
-            metrics.objectSize.update(node.lastQueriedEstimatedSizeOnHeap);
-            maybeEvictSomeNodes();
-            return node;
-        }
-
-        /*
-         * Can't return EVICTED or INITIALIZED
-         */
-        private AccordCachingState<K, V> acquireExisting(AccordCachingState<K, V> node, boolean onlyIfLoaded, @Nullable ExecutorPlus loadExecutor)
-        {
-            Status status = node.status(); // status() completes
-
-            if (status.isLoaded())
-                incrementCacheHits();
-            else
-                incrementCacheMisses();
-
-            if (onlyIfLoaded && !status.isLoaded())
-                return null;
-
-            if (node.references == 0)
-            {
-                if (loadExecutor == null)
-                    loadExecutor = AccordStateCache.this.loadExecutor;
-                if (status == FAILED_TO_LOAD || status == EVICTED)
-                    node.reset().load(loadExecutor, loadFunction);
-
-                if (isInQueue(node))
-                    unlink(node);
-            }
-            node.references++;
-
-            return node;
-        }
-
-        public void release(S safeRef)
-        {
-            K key = safeRef.global().key();
-            logger.trace("Releasing resources for {}: {}", key, safeRef);
-
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-
-            checkState(safeRef.global() != null, "safeRef node is null for %s", key);
-            checkState(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node);
-            checkState(node.references > 0, "references (%d) are zero for %s (%s)", node.references, key, node);
-            checkState(!isInQueue(node));
-
-            if (safeRef.hasUpdate())
-                node.set(safeRef.current());
-            safeRef.invalidate();
-
-            maybeUpdateSize(node, heapEstimator);
-
-            if (listeners != null)
-                listeners.forEach(l -> l.onRelease(node));
-
-            if (--node.references == 0)
-            {
-                Status status = node.status(); // status() completes
-                switch (status)
-                {
-                    default: throw new IllegalStateException("Unhandled status " + status);
-                    case LOADING:
-                    case FAILED_TO_LOAD:
-                        logger.trace("Evicting {} with status {}", key, status);
-                        evict(node);
-                        break;
-                    case LOADED:
-                    case MODIFIED:
-                    case SAVING:
-                        logger.trace("Moving {} with status {} to eviction queue", key, status);
-                        link(node);
-                        break;
-                    case FAILED_TO_SAVE:
-                        break; // can never evict, so no point in adding to eviction queue either
-                }
-            }
-
-            // TODO (performance, expected): triggering on every release is potentially heavy
-            maybeEvictSomeNodes();
-        }
-
-        void validateLoadEvicted(AccordCachingState<?, ?> node)
-        {
-            @SuppressWarnings("unchecked")
-            AccordCachingState<K, V> state = (AccordCachingState<K, V>) node;
-            K key = state.key();
-            V evicted = state.get();
-            if (!validateFunction.apply(key, evicted))
-                throw new IllegalStateException("Reloaded value for key " + key + " is not equal to or fuller than evicted value " + evicted);
-        }
-
-        @VisibleForTesting
-        public AccordCachingState<K, V> getUnsafe(K key)
-        {
-            //noinspection unchecked
-            return (AccordCachingState<K, V>) cache.get(key);
-        }
-
-        @VisibleForTesting
-        public boolean isReferenced(K key)
-        {
-            //noinspection unchecked
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node != null && node.references > 0;
-        }
-
-        @VisibleForTesting
-        public boolean isLoaded(K key)
-        {
-            //noinspection unchecked
-            AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
-            return node != null && node.isLoaded();
-        }
-
-        @VisibleForTesting
-        public boolean hasLoadResult(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() == LOADING;
-        }
-
-        @VisibleForTesting
-        public boolean hasSaveResult(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() == SAVING;
-        }
-
-        @VisibleForTesting
-        public void complete(K key)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            if (node != null)
-                node.complete();
-        }
-
-        @VisibleForTesting
-        boolean keyIsReferenced(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.references > 0;
-        }
-
-        @VisibleForTesting
-        boolean keyIsCached(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null && node.status() != EVICTED;
-        }
-
-        @VisibleForTesting
-        int references(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
-        {
-            AccordCachingState<?, ?> node = cache.get(key);
-            return node != null ? node.references : 0;
-        }
-
-        private void incrementCacheQueries()
-        {
-            instanceMetrics.requests.mark();
-            metrics.requests.mark();
-            stats.queries++;
-            AccordStateCache.this.stats.queries++;
-        }
-
-        private void incrementCacheHits()
-        {
-            instanceMetrics.hits.mark();
-            metrics.hits.mark();
-            stats.hits++;
-            AccordStateCache.this.stats.hits++;
-        }
-
-        private void incrementCacheMisses()
-        {
-            instanceMetrics.misses.mark();
-            metrics.misses.mark();
-            stats.misses++;
-            AccordStateCache.this.stats.misses++;
-        }
-
-        public Stats stats()
-        {
-            return stats;
-        }
-
-        public ImmutableStats statsSnapshot()
-        {
-            return new ImmutableStats(stats);
-        }
-
-        public Stats globalStats()
-        {
-            return AccordStateCache.this.stats;
-        }
-
-        @VisibleForTesting
-        public void unsafeSetLoadFunction(Function<K, V> loadFunction)
-        {
-            this.loadFunction = loadFunction;
-        }
-
-        @VisibleForTesting
-        public void unsafeSetSaveFunction(Function<V, Runnable> saveFunction)
-        {
-            this.saveFunction = saveFunction;
-        }
-
-        @Override
-        public long capacity()
-        {
-            return AccordStateCache.this.capacity();
-        }
-
-        @Override
-        public void setCapacity(long capacity)
-        {
-            throw new UnsupportedOperationException("Capacity is shared between all instances. Please set the capacity on the global cache");
-        }
-
-        @Override
-        public int size()
-        {
-            return cache.size();
-        }
-
-        @Override
-        public long weightedSize()
-        {
-            return bytesCached;
-        }
-
-        public long globalAllocated()
-        {
-            return AccordStateCache.this.bytesCached;
-        }
-
-        public int globalReferencedEntries()
-        {
-            return AccordStateCache.this.numReferencedEntries();
-        }
-
-        public int globalUnreferencedEntries()
-        {
-            return AccordStateCache.this.numUnreferencedEntries();
-        }
-
-        @Override
-        public String toString()
-        {
-            return "Instance{" +
-                   "index=" + index +
-                   ", keyClass=" + keyClass +
-                   '}';
-        }
-    }
-
-    @VisibleForTesting
-    void unsafeClear()
-    {
-        bytesCached = 0;
-        metrics.reset();;
-        instances.values().forEach(instance -> {
-            instance.cache.forEach((k, v) -> Invariants.checkState(v.references == 0));
-            instance.cache.clear();
-            instance.bytesCached = 0;
-            instance.instanceMetrics.reset();
-        });
-        //noinspection StatementWithEmptyBody
-        while (null != poll());
-    }
-
-    @VisibleForTesting
-    AccordCachingState<?, ?> head()
-    {
-        Iterator<AccordCachingState<?, ?>> iter = iterator();
-        return iter.hasNext() ? iter.next() : null;
-    }
-
-    @VisibleForTesting
-    AccordCachingState<?, ?> tail()
-    {
-        AccordCachingState<?,?> last = null;
-        Iterator<AccordCachingState<?, ?>> iter = iterator();
-        while (iter.hasNext())
-            last = iter.next();
-        return last;
-    }
-
-    @VisibleForTesting
-    public void awaitSaveResults()
-    {
-        for (AccordCachingState<?, ?> node : this)
-            if (node.status() == SAVING)
-                AsyncChains.awaitUninterruptibly(node.saving());
-    }
-
-    private int cacheSize()
-    {
-        int size = 0;
-        for (Instance<?, ?, ?> instance : instances.values())
-            size += instance.cache.size();
-        return size;
-    }
-
-    @VisibleForTesting
-    int numReferencedEntries()
-    {
-        return cacheSize() - unreferenced;
-    }
-
-    @VisibleForTesting
-    int numUnreferencedEntries()
-    {
-        return unreferenced;
-    }
-
-    @Override
-    public int size()
-    {
-        return cacheSize();
-    }
-
-    @Override
-    public long weightedSize()
-    {
-        return bytesCached;
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/accord/AccordTask.java b/src/java/org/apache/cassandra/service/accord/AccordTask.java
new file mode 100644
index 0000000000..b7d3cd25ce
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/accord/AccordTask.java
@@ -0,0 +1,1114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import accord.api.RoutingKey;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommandStore;
+import accord.local.cfk.CommandsForKey;
+import accord.primitives.AbstractRanges;
+import accord.primitives.AbstractUnseekableKeys;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.Cancellable;
+import org.agrona.collections.Object2ObjectHashMap;
+import org.agrona.collections.ObjectHashSet;
+import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
+import org.apache.cassandra.service.accord.AccordCommandStore.Caches;
+import org.apache.cassandra.service.accord.AccordExecutor.Task;
+import org.apache.cassandra.service.accord.AccordExecutor.TaskQueue;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.NoSpamLogger;
+import org.apache.cassandra.utils.concurrent.Condition;
+
+import static accord.local.KeyHistory.TIMESTAMPS;
+import static accord.primitives.Routable.Domain.Key;
+import static accord.primitives.Txn.Kind.EphemeralRead;
+import static accord.utils.Invariants.illegalState;
+import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED;
+import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED;
+import static org.apache.cassandra.service.accord.AccordTask.State.FAILED;
+import static org.apache.cassandra.service.accord.AccordTask.State.FAILING;
+import static org.apache.cassandra.service.accord.AccordTask.State.FINISHED;
+import static org.apache.cassandra.service.accord.AccordTask.State.INITIALIZED;
+import static org.apache.cassandra.service.accord.AccordTask.State.LOADING;
+import static org.apache.cassandra.service.accord.AccordTask.State.PERSISTING;
+import static org.apache.cassandra.service.accord.AccordTask.State.RUNNING;
+import static org.apache.cassandra.service.accord.AccordTask.State.SCANNING_RANGES;
+import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_LOAD;
+import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_RUN;
+import static org.apache.cassandra.service.accord.AccordTask.State.WAITING_TO_SCAN_RANGES;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+public abstract class AccordTask<R> extends Task implements Runnable, Function<SafeCommandStore, R>, Cancellable
+{
+    private static final Logger logger = LoggerFactory.getLogger(AccordTask.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+    private static final boolean SANITY_CHECK = DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED.getBoolean();
+
+    private static class LoggingProps
+    {
+        private static final String COMMAND_STORE = "command_store";
+        private static final String ACCORD_TASK = "accord_task";
+    }
+
+    static class ForFunction<R> extends AccordTask<R>
+    {
+        private final Function<? super SafeCommandStore, R> function;
+
+        public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function<? super SafeCommandStore, R> function)
+        {
+            super(commandStore, loadCtx);
+            this.function = function;
+        }
+
+        @Override
+        public R apply(SafeCommandStore commandStore)
+        {
+            return function.apply(commandStore);
+        }
+    }
+
+    // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names.
+    static class ForConsumer extends AccordTask<Void>
+    {
+        private final Consumer<? super SafeCommandStore> consumer;
+
+        private ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer<? super SafeCommandStore> consumer)
+        {
+            super(commandStore, loadCtx);
+            this.consumer = consumer;
+        }
+
+        @Override
+        public Void apply(SafeCommandStore commandStore)
+        {
+            consumer.accept(commandStore);
+            return null;
+        }
+    }
+
+    public static <T> AccordTask<T> create(CommandStore commandStore, PreLoadContext ctx, Function<? super SafeCommandStore, T> function)
+    {
+        return new ForFunction<>((AccordCommandStore) commandStore, ctx, function);
+    }
+
+    public static AccordTask<Void> create(CommandStore commandStore, PreLoadContext ctx, Consumer<? super SafeCommandStore> consumer)
+    {
+        return new ForConsumer((AccordCommandStore) commandStore, ctx, consumer);
+    }
+
+    public enum State
+    {
+        INITIALIZED(),
+        WAITING_TO_SCAN_RANGES(INITIALIZED),
+        SCANNING_RANGES(WAITING_TO_SCAN_RANGES),
+        WAITING_TO_LOAD(INITIALIZED, SCANNING_RANGES),
+        LOADING(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD),
+        WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING),
+        RUNNING(WAITING_TO_RUN),
+        PERSISTING(RUNNING),
+        FAILING(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, RUNNING, PERSISTING),
+        FINISHED(RUNNING, PERSISTING),
+        CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN),
+        FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, RUNNING, PERSISTING, FAILING);
+
+        private final int permittedFrom;
+
+        State()
+        {
+            this.permittedFrom = 0;
+        }
+
+        State(State ... permittedFroms)
+        {
+            int permittedFrom = 0;
+            for (State state : permittedFroms)
+                permittedFrom |= 1 << state.ordinal();
+            this.permittedFrom = permittedFrom;
+        }
+
+        boolean isPermittedFrom(State prev)
+        {
+            return (permittedFrom & (1 << prev.ordinal())) != 0;
+        }
+
+        boolean isExecuted()
+        {
+            return this.compareTo(PERSISTING) >= 0;
+        }
+
+        boolean isComplete()
+        {
+            return this.compareTo(FINISHED) >= 0;
+        }
+    }
+
+    private State state = INITIALIZED;
+    private final PreLoadContext preLoadContext;
+    private final String loggingId;
+
+    @Nullable Object2ObjectHashMap<TxnId, AccordSafeCommand> commands;
+    @Nullable Object2ObjectHashMap<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey;
+    @Nullable Object2ObjectHashMap<RoutingKey, AccordSafeCommandsForKey> commandsForKey;
+    @Nullable Object2ObjectHashMap<Object, AccordSafeState<?, ?>> loading;
+    // TODO (expected): collection supporting faster deletes but still fast poll (e.g. some ordered collection)
+    @Nullable ArrayDeque<AccordCacheEntry<?, ?>> waitingToLoad;
+    @Nullable RangeScanner rangeScanner;
+    @Nullable CommandsForRanges commandsForRanges;
+    @Nullable private TaskQueue queued;
+
+    private BiConsumer<? super R, Throwable> callback;
+    private List<Command> sanityCheck;
+    public long createdAt = nanoTime(), loadedAt, runQueuedAt, runAt, completedAt;
+
+    private void setLoggingIds()
+    {
+        MDC.put(LoggingProps.COMMAND_STORE, commandStore.loggingId);
+        MDC.put(LoggingProps.ACCORD_TASK, loggingId);
+    }
+
+    private void clearLoggingIds()
+    {
+        MDC.remove(LoggingProps.COMMAND_STORE);
+        MDC.remove(LoggingProps.ACCORD_TASK);
+    }
+
+    public AccordTask(AccordCommandStore commandStore, PreLoadContext preLoadContext)
+    {
+        super(commandStore);
+        this.loggingId = "0x" + Integer.toHexString(System.identityHashCode(this));
+        this.preLoadContext = preLoadContext;
+
+        if (logger.isTraceEnabled())
+        {
+            setLoggingIds();
+            logger.trace("Created {} on {}", this, commandStore);
+            clearLoggingIds();
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return "AccordTask{" + state + "}-" + loggingId;
+    }
+
+    public String toDescription()
+    {
+        return "AccordTask{" + state + "}-" + loggingId + ": "
+               + (queued == null ? "unqueued" : queued.kind)
+               + ", primaryTxnId: " + preLoadContext.primaryTxnId()
+               + ", waitingToLoad: " + summarise(waitingToLoad)
+               + ", loading:" + summarise(loading, AccordSafeState::global)
+               + ", cfks:" + summarise(commandsForKey, AccordSafeState::global)
+               + ", tfks:" + summarise(timestampsForKey, AccordSafeState::global)
+               + ", txns:" + summarise(commands, AccordSafeState::global);
+
+    }
+
+    private static <V> String summarise(Map<?, V> map, Function<V, Object> transform)
+    {
+        if (map == null)
+            return "null";
+
+        return summarise(map.values(), transform);
+    }
+
+    private static <V> String summarise(Collection<V> collection)
+    {
+        return summarise(collection, Function.identity());
+    }
+
+    private static <V> String summarise(Collection<V> collection, Function<? super V, Object> transform)
+    {
+        if (collection == null)
+            return "null";
+
+        StringBuilder out = new StringBuilder("[");
+        int count = 0;
+        for (V v : collection)
+        {
+            if (count++ > 0)
+            {
+                out.append(',');
+                if (count >= 10)
+                {
+                    out.append("...(*").append(collection.size() - 10).append(')');
+                    break;
+                }
+            }
+            out.append(transform.apply(v));
+        }
+        out.append(']');
+        return out.toString();
+    }
+
+    private void state(State state)
+    {
+        Invariants.checkState(state.isPermittedFrom(this.state), "%s forbidden from %s", state, this, AccordTask::toDescription);
+        this.state = state;
+        if (state == WAITING_TO_RUN)
+        {
+            Invariants.checkState(loading == null && waitingToLoad == null, "WAITING_TO_RUN => no loading or waiting; found %s", this, AccordTask::toDescription);
+            loadedAt = nanoTime();
+        }
+        else if (state == RUNNING)
+        {
+            runAt = nanoTime();
+        }
+        else if (state.isExecuted())
+        {
+            completedAt = nanoTime();
+        }
+    }
+
+    Unseekables<?> keys()
+    {
+        return preLoadContext.keys();
+    }
+
+    public AsyncChain<R> chain()
+    {
+        return new AsyncChains.Head<>()
+        {
+            @Override
+            protected Cancellable start(BiConsumer<? super R, Throwable> callback)
+            {
+                Invariants.checkState(AccordTask.this.callback == null);
+                AccordTask.this.callback = callback;
+                commandStore.tryPreSetup(AccordTask.this);
+                commandStore.executor().submit(AccordTask.this);
+                return AccordTask.this;
+            }
+        };
+    }
+
+    // to be invoked only by the CommandStore owning thread, to take references to objects already in use by the current execution
+    public void presetup(AccordTask<?> parent)
+    {
+        // note we use the caches "unsafely" here deliberately, as we only reference commands we already have references to
+        // so we do not mutate anything, except the atomic counter of references
+        if (parent.commands != null)
+        {
+            for (TxnId txnId : preLoadContext.txnIds())
+                presetupExclusive(txnId, AccordTask::ensureCommands, parent.commands, commandStore.cachesUnsafe().commands());
+        }
+
+        if ((preLoadContext.keyHistory() == TIMESTAMPS ? parent.timestampsForKey : parent.commandsForKey) == null) return;
+        if (preLoadContext.keys().domain() != Key) return;
+        switch (preLoadContext.keyHistory())
+        {
+            default: throw new AssertionError("Unhandled KeyHistory: " + preLoadContext.keyHistory());
+            case NONE:
+                break;
+
+            case TIMESTAMPS:
+                for (RoutingKey key : (AbstractUnseekableKeys)preLoadContext.keys())
+                    presetupExclusive(key, AccordTask::ensureTimestampsForKey, parent.timestampsForKey, commandStore.cachesUnsafe().timestampsForKeys());
+                break;
+
+            case ASYNC:
+            case RECOVER:
+            case INCR:
+            case SYNC:
+                for (RoutingKey key : (AbstractUnseekableKeys)preLoadContext.keys())
+                    presetupExclusive(key, AccordTask::ensureCommandsForKey, parent.commandsForKey, commandStore.cachesUnsafe().commandsForKeys());
+                break;
+        }
+    }
+
+    public void setupExclusive()
+    {
+        setupInternal(commandStore.cachesExclusive());
+        state(rangeScanner != null ? WAITING_TO_SCAN_RANGES
+                                   : waitingToLoad != null ? State.WAITING_TO_LOAD
+                                    : loading != null ? LOADING : WAITING_TO_RUN);
+    }
+
+    private void setupInternal(Caches caches)
+    {
+        {
+            boolean hasPreSetup = commands != null;
+            for (TxnId txnId : preLoadContext.txnIds())
+            {
+                if (hasPreSetup && completePresetupExclusive(txnId, commands, caches.commands()))
+                    continue;
+                setupExclusive(txnId, AccordTask::ensureCommands, caches.commands());
+            }
+        }
+
+        if (preLoadContext.keys().isEmpty())
+            return;
+
+        switch (preLoadContext.keys().domain())
+        {
+            case Key: setupKeyLoadsExclusive(caches, (AbstractUnseekableKeys)preLoadContext.keys()); break;
+            case Range: setupRangeLoadsExclusive(caches);
+        }
+    }
+
+    private void setupKeyLoadsExclusive(Caches caches, Iterable<? extends RoutingKey> keys)
+    {
+        switch (preLoadContext.keyHistory())
+        {
+            default: throw new AssertionError("Unhandled KeyHistory: " + preLoadContext.keyHistory());
+            case NONE:
+                break;
+
+            case TIMESTAMPS:
+            {
+                boolean hasPreSetup = timestampsForKey != null;
+                for (RoutingKey key : keys)
+                {
+                    if (hasPreSetup && completePresetupExclusive(key, timestampsForKey, caches.timestampsForKeys())) continue;
+                    setupExclusive(key, AccordTask::ensureTimestampsForKey, caches.timestampsForKeys());
+                }
+                break;
+            }
+            case ASYNC:
+            case RECOVER:
+            case INCR:
+            case SYNC:
+            {
+                boolean hasPreSetup = commandsForKey != null;
+                for (RoutingKey key : keys)
+                {
+                    if (hasPreSetup && completePresetupExclusive(key, commandsForKey, caches.commandsForKeys())) continue;
+                    setupExclusive(key, AccordTask::ensureCommandsForKey, caches.commandsForKeys());
+                }
+                break;
+            }
+        }
+    }
+
+    private void setupRangeLoadsExclusive(Caches caches)
+    {
+        switch (preLoadContext.keyHistory())
+        {
+            default: throw new AssertionError("Unhandled KeyHistory: " + preLoadContext.keyHistory());
+            case NONE:
+            case ASYNC:
+                break;
+
+            case INCR:
+                throw new AssertionError("Incremental mode should only be used with an explicit list of keys");
+
+            case TIMESTAMPS:
+                throw new AssertionError("TimestampsForKey unsupported for range transactions");
+
+            case RECOVER:
+            case SYNC:
+                rangeScanner = new RangeScanner(caches.commandsForKeys());
+        }
+    }
+
+    // expects mutual exclusivity only on the command store
+    private <K, V, S extends AccordSafeState<K, V>> void presetupExclusive(K k, Function<AccordTask<?>, Map<? super K, ? super S>> loaded, Map<? super K, S> parentMap, AccordCache.Type<K, V, S>.Instance cache)
+    {
+        AccordSafeState<K, V> ref = parentMap.get(k);
+        if (ref == null)
+            return;
+
+        AccordCacheEntry<K, V> node = ref.global();
+        int refs = node.increment();
+        Invariants.checkState(refs > 1);
+        loaded.apply(this).put(k, cache.parent().adapter().safeRef(node));
+    }
+
+    // expects to hold lock
+    private <K, V, S extends AccordSafeState<K, V>> boolean completePresetupExclusive(K k, Map<? super K, S> map, AccordCache.Type<K, V, S>.Instance cache)
+    {
+        AccordSafeState<K, V> preacquired = map.get(k);
+        if (preacquired != null)
+        {
+            cache.recordPreAcquired(preacquired);
+            return true;
+        }
+        return false;
+    }
+
+    // expects to hold lock
+    private <K, V, S extends AccordSafeState<K, V>> void setupExclusive(K k, Function<AccordTask<?>, Map<? super K, ? super S>> loaded, AccordCache.Type<K, V, S>.Instance cache)
+    {
+        S safeRef = cache.acquire(k);
+        Status entryStatus = safeRef.global().status();
+        Map<? super K, ? super S> map;
+        switch (entryStatus)
+        {
+            default: throw new IllegalStateException("Unhandled global state: " + entryStatus);
+            case WAITING_TO_LOAD:
+            case LOADING:
+                map = ensureLoading();
+                break;
+            case SAVING:
+            case LOADED:
+            case MODIFIED:
+            case FAILED_TO_SAVE:
+                map = loaded.apply(this);
+        }
+
+        Object prev = map.putIfAbsent(k, safeRef);
+        if (prev != null)
+        {
+            noSpamLogger.warn("PreLoadContext {} contained key {} more than once", map, k);
+            cache.release(safeRef, this);
+        }
+        else if (map == loading)
+        {
+            if (entryStatus == Status.WAITING_TO_LOAD)
+                ensureWaitingToLoad().add(safeRef.global());
+            safeRef.global().loadingOrWaiting().add(this);
+            Invariants.paranoid(safeRef.global().loadingOrWaiting().waiters().size() == safeRef.global().references());
+        }
+    }
+
+    // expects to hold lock
+    public boolean onLoad(AccordCacheEntry<?, ?> state)
+    {
+        AccordSafeState<?, ?> safeRef = loading == null ? null : loading.remove(state.key());
+        Invariants.checkState(safeRef != null && safeRef.global() == state, "Expected to find %s loading; found %s", state, this, AccordTask::toDescription);
+        if (safeRef.getClass() == AccordSafeCommand.class)
+        {
+            ensureCommands().put((TxnId)state.key(), (AccordSafeCommand) safeRef);
+        }
+        else if (safeRef.getClass() == AccordSafeCommandsForKey.class)
+        {
+            ensureCommandsForKey().put((RoutingKey) state.key(), (AccordSafeCommandsForKey) safeRef);
+        }
+        else
+        {
+            Invariants.checkState (safeRef.getClass() == AccordSafeTimestampsForKey.class);
+            ensureTimestampsForKey().put((RoutingKey) state.key(), (AccordSafeTimestampsForKey) safeRef);
+        }
+
+        if (!loading.isEmpty())
+            return false;
+
+        loading = null;
+        if (this.state.compareTo(State.WAITING_TO_LOAD) < 0)
+            return false;
+
+        Invariants.checkState(waitingToLoad == null, "Invalid state: %s", this, AccordTask::toDescription);
+        state(WAITING_TO_RUN);
+        return true;
+    }
+
+    // expects to hold lock
+    public boolean onLoading(AccordCacheEntry<?, ?> state)
+    {
+        boolean removed = waitingToLoad != null && waitingToLoad.remove(state);
+        Invariants.checkState(removed, "%s not found in waitingToLoad %s", state, this, AccordTask::toDescription);
+        if (!waitingToLoad.isEmpty())
+            return false;
+
+        return onEmptyWaitingToLoad();
+    }
+
+    private boolean onEmptyWaitingToLoad()
+    {
+        waitingToLoad = null;
+        if (this.state.compareTo(State.WAITING_TO_LOAD) < 0)
+            return false;
+
+        state(loading == null ? WAITING_TO_RUN : LOADING);
+        return true;
+    }
+
+    public PreLoadContext preLoadContext()
+    {
+        return preLoadContext;
+    }
+
+    public Map<TxnId, AccordSafeCommand> commands()
+    {
+        return commands;
+    }
+
+    public Map<TxnId, AccordSafeCommand> ensureCommands()
+    {
+        if (commands == null)
+            commands = new Object2ObjectHashMap<>();
+        return commands;
+    }
+
+    public Map<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey()
+    {
+        return timestampsForKey;
+    }
+
+    public Map<RoutingKey, AccordSafeTimestampsForKey> ensureTimestampsForKey()
+    {
+        if (timestampsForKey == null)
+            timestampsForKey = new Object2ObjectHashMap<>();
+        return timestampsForKey;
+    }
+
+    public Map<RoutingKey, AccordSafeCommandsForKey> commandsForKey()
+    {
+        return commandsForKey;
+    }
+
+    public Map<RoutingKey, AccordSafeCommandsForKey> ensureCommandsForKey()
+    {
+        if (commandsForKey == null)
+            commandsForKey = new Object2ObjectHashMap<>();
+        return commandsForKey;
+    }
+
+    private Map<Object, AccordSafeState<?, ?>> ensureLoading()
+    {
+        if (loading == null)
+            loading = new Object2ObjectHashMap<>();
+        return loading;
+    }
+
+    private ArrayDeque<AccordCacheEntry<?, ?>> ensureWaitingToLoad()
+    {
+        Invariants.checkState(state.compareTo(WAITING_TO_LOAD) <= 0, "Expected status to be on or before WAITING_TO_LOAD; found %s", this, AccordTask::toDescription);
+        if (waitingToLoad == null)
+            waitingToLoad = new ArrayDeque<>();
+        return waitingToLoad;
+    }
+
+    public AccordCacheEntry<?, ?> pollWaitingToLoad()
+    {
+        Invariants.checkState(state == State.WAITING_TO_LOAD, "Expected status to be WAITING_TO_LOAD; found %s", this, AccordTask::toDescription);
+        if (waitingToLoad == null)
+            return null;
+
+        AccordCacheEntry<?, ?> next = waitingToLoad.poll();
+        if (waitingToLoad.isEmpty())
+            onEmptyWaitingToLoad();
+        return next;
+    }
+
+    public AccordCacheEntry<?, ?> peekWaitingToLoad()
+    {
+        return waitingToLoad == null ? null : waitingToLoad.peek();
+    }
+
+    private void maybeSanityCheck(AccordSafeCommand safeCommand)
+    {
+        if (SANITY_CHECK)
+        {
+            if (sanityCheck == null)
+                sanityCheck = new ArrayList<>(commands.size());
+            sanityCheck.add(safeCommand.current());
+        }
+    }
+
+    private void save(List<SavedCommand.Writer> diffs, Runnable onFlush)
+    {
+        if (sanityCheck != null)
+        {
+            Invariants.checkState(SANITY_CHECK);
+            Condition condition = Condition.newOneTimeCondition();
+            this.commandStore.appendCommands(diffs, condition::signal);
+            condition.awaitUninterruptibly();
+
+            for (Command check : sanityCheck)
+                this.commandStore.sanityCheckCommand(check);
+
+            if (onFlush != null) onFlush.run();
+        }
+        else
+        {
+            this.commandStore.appendCommands(diffs, onFlush);
+        }
+    }
+
+    @Override
+    protected void preRunExclusive()
+    {
+        state(RUNNING);
+        if (rangeScanner != null)
+        {
+            commandsForRanges = rangeScanner.finish(commandStore.cachesExclusive());
+            rangeScanner = null;
+        }
+        if (commands != null)
+            commands.forEach((k, v) -> v.preExecute());
+        if (commandsForKey != null)
+            commandsForKey.forEach((k, v) -> v.preExecute());
+        if (timestampsForKey != null)
+            timestampsForKey.forEach((k, v) -> v.preExecute());
+    }
+
+    @Override
+    public void run()
+    {
+        setLoggingIds();
+        logger.trace("Running {} with state {}", this, state);
+        AccordSafeCommandStore safeStore = null;
+        try
+        {
+            if (state != RUNNING)
+                throw illegalState("Unexpected state " + toDescription());
+
+            safeStore = commandStore.begin(this, commandsForRanges);
+            R result = apply(safeStore);
+
+            // TODO (required): currently, we are not very efficient about ensuring that we persist the absolute minimum amount of state. Improve that.
+            List<SavedCommand.Writer> diffs = null;
+            if (commands != null)
+            {
+                for (AccordSafeCommand safeCommand : commands.values())
+                {
+                    if (safeCommand.txnId().is(EphemeralRead))
+                        continue;
+
+                    SavedCommand.Writer diff = safeCommand.diff();
+                    if (diff == null)
+                        continue;
+
+                    if (diffs == null)
+                        diffs = new ArrayList<>(commands.size());
+                    diffs.add(diff);
+
+                    maybeSanityCheck(safeCommand);
+                }
+            }
+
+            boolean flush = diffs != null || safeStore.fieldUpdates() != null;
+            if (flush)
+            {
+                state(PERSISTING);
+                Runnable onFlush = () -> finish(result, null);
+                if (safeStore.fieldUpdates() != null)
+                    commandStore.persistFieldUpdates(safeStore.fieldUpdates(), diffs == null ? onFlush : null);
+                if (diffs != null)
+                    save(diffs, onFlush);
+            }
+
+            commandStore.complete(safeStore);
+            safeStore = null;
+            if (!flush)
+                finish(result, null);
+        }
+        catch (Throwable t)
+        {
+            if (safeStore != null)
+            {
+                revert();
+                commandStore.abort(safeStore);
+            }
+            throw t;
+        }
+        finally
+        {
+            logger.trace("Exiting {}", this);
+            clearLoggingIds();
+        }
+    }
+
+    public void fail(Throwable throwable)
+    {
+        commandStore.agent().onUncaughtException(throwable);
+        if (state.isComplete())
+            return;
+
+        if (commandStore.hasSafeStore())
+            commandStore.agent().onUncaughtException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
+
+        state(FAILING);
+        if (callback != null)
+            callback.accept(null, throwable);
+    }
+
+    public void failExclusive(Throwable throwable)
+    {
+        boolean newFailure = state != FAILING;
+        try
+        {
+            if (newFailure)
+            {
+                commandStore.agent().onUncaughtException(throwable);
+                if (state.isComplete())
+                    return;
+
+                if (commandStore.hasSafeStore())
+                    commandStore.agent().onUncaughtException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
+            }
+
+            state(FAILED);
+        }
+        finally
+        {
+            if (newFailure && callback != null)
+                callback.accept(null, throwable);
+        }
+    }
+
+    protected void cleanupExclusive()
+    {
+        releaseResources(commandStore.cachesExclusive());
+        if (state == FAILING)
+            state(FAILED);
+    }
+
+    public RangeScanner rangeScanner()
+    {
+        return rangeScanner;
+    }
+
+    public boolean hasRanges()
+    {
+        return rangeScanner != null;
+    }
+
+    @Override
+    public void cancel()
+    {
+        commandStore.executor().cancel(this);
+    }
+
+    public void cancelExclusive()
+    {
+        releaseResources(commandStore.cachesExclusive());
+        state(CANCELLED);
+        if (callback != null)
+            callback.accept(null, new CancellationException());
+    }
+
+    public State state()
+    {
+        return state;
+    }
+
+    private void finish(R result, Throwable failure)
+    {
+        state(failure == null ? FINISHED : FAILED);
+        if (callback != null)
+            callback.accept(result, failure);
+    }
+
+    void releaseResources(Caches caches)
+    {
+        try
+        {
+            // TODO (expected): we should destructively iterate to avoid invoking second time in fail; or else read and set to null
+            if (rangeScanner != null)
+            {
+                caches.commands().tryUnregister(rangeScanner.commandWatcher);
+                caches.commandsForKeys().tryUnregister(rangeScanner.keyWatcher);
+                rangeScanner = null;
+            }
+            if (commands != null)
+            {
+                commands.forEach((k, v) -> caches.commands().release(v, this));
+                commands.clear();
+                commands = null;
+            }
+            if (timestampsForKey != null)
+            {
+                timestampsForKey.forEach((k, v) -> caches.timestampsForKeys().release(v, this));
+                timestampsForKey.clear();
+                timestampsForKey = null;
+            }
+            if (commandsForKey != null)
+            {
+                commandsForKey.forEach((k, v) -> caches.commandsForKeys().release(v, this));
+                commandsForKey.clear();
+                commandsForKey = null;
+            }
+            if (waitingToLoad != null)
+            {
+                while (!waitingToLoad.isEmpty())
+                    waitingToLoad.poll().loadingOrWaiting().remove(this);
+                waitingToLoad = null;
+            }
+            if (loading != null)
+            {
+                loading.forEach((k, v) -> caches.global().release(v, this));
+                loading.clear();
+                loading = null;
+            }
+        }
+        catch (Throwable t)
+        {
+            releaseResourcesSlow(caches, t);
+            throw t;
+        }
+    }
+
+    private void releaseResourcesSlow(Caches caches, Throwable suppressedBy)
+    {
+        if (commands != null)
+        {
+            safeRelease(commands, caches.commands(), suppressedBy);
+            commands.clear();
+            commands = null;
+        }
+        if (timestampsForKey != null)
+        {
+            safeRelease(timestampsForKey, caches.timestampsForKeys(), suppressedBy);
+            timestampsForKey.clear();
+            timestampsForKey = null;
+        }
+        if (commandsForKey != null)
+        {
+            safeRelease(commandsForKey, caches.commandsForKeys(), suppressedBy);
+            commandsForKey.clear();
+            commandsForKey = null;
+        }
+        if (waitingToLoad != null)
+        {
+            while (!waitingToLoad.isEmpty())
+            {
+                try { waitingToLoad.poll().loadingOrWaiting().remove(this); }
+                catch (Throwable t) { suppressedBy.addSuppressed(t); }
+            }
+            waitingToLoad = null;
+        }
+        if (loading != null)
+        {
+            safeRelease(loading, caches.global(), suppressedBy);
+            loading.clear();
+            loading = null;
+        }
+    }
+
+    private <K, V> void safeRelease(Map<K, ? extends AccordSafeState<K, V>> map, AccordCache.Type<K, V, ?>.Instance cache, Throwable suppressedBy)
+    {
+        for (AccordSafeState<K, V> safeState : map.values())
+        {
+            if (safeState.invalidated()) continue;
+            try { cache.release(safeState, this); }
+            catch (Throwable t) { suppressedBy.addSuppressed(t); }
+        }
+    }
+
+    private void safeRelease(Map<?, ? extends AccordSafeState<?, ?>> map, AccordCache cache, Throwable suppressedBy)
+    {
+        for (AccordSafeState<?, ?> safeState : map.values())
+        {
+            if (safeState.invalidated()) continue;
+            try { cache.release(safeState, this); }
+            catch (Throwable t) { suppressedBy.addSuppressed(t); }
+        }
+    }
+
+    void revert()
+    {
+        if (commands != null)
+            commands.forEach((k, v) -> v.revert());
+        if (timestampsForKey != null)
+            timestampsForKey.forEach((k, v) -> v.revert());
+        if (commandsForKey != null)
+            commandsForKey.forEach((k, v) -> v.revert());
+    }
+
+    public class RangeScanner implements Runnable
+    {
+        class KeyWatcher implements AccordCache.Listener<RoutingKey, CommandsForKey>
+        {
+            @Override
+            public void onUpdate(AccordCacheEntry<RoutingKey, CommandsForKey> state)
+            {
+                if (ranges.contains(state.key()))
+                    reference(state);
+            }
+        }
+
+        class CommandWatcher implements AccordCache.Listener<TxnId, Command>
+        {
+            @Override
+            public void onUpdate(AccordCacheEntry<TxnId, Command> state)
+            {
+                CommandsForRangesLoader.Summary summary = summaryLoader.from(state);
+                if (summary != null)
+                    summaries.put(summary.txnId, summary);
+            }
+        }
+
+        final ConcurrentHashMap<TxnId, CommandsForRangesLoader.Summary> summaries = new ConcurrentHashMap<>();
+        // TODO (expected): produce key summaries to avoid locking all in memory
+        final Set<AccordRoutingKey.TokenKey> intersectingKeys = new ObjectHashSet<>();
+        final KeyWatcher keyWatcher = new KeyWatcher();
+        final CommandWatcher commandWatcher = new CommandWatcher();
+        final Ranges ranges = ((AbstractRanges) preLoadContext.keys()).toRanges();
+        final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeyCache;
+
+        public RangeScanner(AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeyCache)
+        {
+            this.commandsForKeyCache = commandsForKeyCache;
+        }
+
+        CommandsForRangesLoader.Loader summaryLoader;
+        boolean scanned;
+
+        @Override
+        public void run()
+        {
+            try
+            {
+                for (Range range : ranges)
+                {
+                    AccordKeyspace.findAllKeysBetween(commandStore.id(),
+                                                      (AccordRoutingKey) range.start(), range.startInclusive(),
+                                                      (AccordRoutingKey) range.end(), range.endInclusive(),
+                                                      intersectingKeys::add);
+                }
+
+                Collection<TxnId> txnIds = summaryLoader.intersects();
+                for (TxnId txnId : txnIds)
+                {
+                    if (summaries.containsKey(txnId))
+                        continue;
+
+                    CommandsForRangesLoader.Summary summary = summaryLoader.load(txnId);
+                    if (summary != null)
+                        summaries.putIfAbsent(txnId, summary);
+                }
+            }
+            catch (Throwable t)
+            {
+                commandStore.executor().onScannedRanges(AccordTask.this, t);
+                throw t;
+            }
+            commandStore.executor().onScannedRanges(AccordTask.this, null);
+        }
+
+        private void reference(AccordCacheEntry<RoutingKey, CommandsForKey> entry)
+        {
+            if (loading != null && loading.containsKey(entry.key()))
+                return;
+
+            switch (entry.status())
+            {
+                default: throw new AssertionError("Unhandled Status: " + entry.status());
+                case WAITING_TO_LOAD:
+                case LOADING:
+                    if (scanned)
+                        // if we've finished scanning and not already taken a reference we shouldn't need to witness (unless modified)
+                        return;
+                    ensureLoading().put(entry.key(), commandsForKeyCache.acquire(entry));
+                    if (entry.status() == Status.WAITING_TO_LOAD)
+                        ensureWaitingToLoad().add(entry);
+                    entry.loadingOrWaiting().add(AccordTask.this);
+                    return;
+
+                case MODIFIED:
+                case SAVING:
+                case LOADED:
+                case FAILED_TO_SAVE:
+                    if (commandsForKey != null && commandsForKey.containsKey(entry.key()))
+                        return;
+                    ensureCommandsForKey().putIfAbsent(entry.key(), commandsForKeyCache.acquire(entry));
+            }
+        }
+
+        public void start(BiFunction<Task, Runnable, Cancellable> executor)
+        {
+            Caches caches = commandStore.cachesExclusive();
+            state(SCANNING_RANGES);
+
+            for (RoutingKey key : caches.commandsForKeys().keySet())
+            {
+                if (ranges.contains(key))
+                    intersectingKeys.add((AccordRoutingKey.TokenKey) key);
+            }
+
+            summaryLoader = commandStore.diskCommandsForRanges().loader(preLoadContext.primaryTxnId(), preLoadContext.keyHistory(), ranges);
+            summaryLoader.forEachInCache(summary -> summaries.put(summary.txnId, summary), caches);
+            caches.commandsForKeys().register(keyWatcher);
+            caches.commands().register(commandWatcher);
+            // TODO (expected): support cancellation here
+            executor.apply(AccordTask.this, this);
+        }
+
+        public void scannedExclusive()
+        {
+            Invariants.checkState(state == SCANNING_RANGES, "Expected SCANNING_RANGES; found %s", AccordTask.this, AccordTask::toDescription);
+            scanned = true;
+
+            if (commandsForKey != null)
+                intersectingKeys.removeAll(commandsForKey.keySet());
+            if (loading != null)
+                intersectingKeys.removeAll(loading.keySet());
+            setupKeyLoadsExclusive(commandStore.cachesExclusive(), intersectingKeys);
+
+            if (loading == null) state(WAITING_TO_RUN);
+            else if (waitingToLoad == null) state(LOADING);
+            else state(State.WAITING_TO_LOAD);
+        }
+
+        CommandsForRanges finish(Caches caches)
+        {
+            caches.commandsForKeys().unregister(keyWatcher);
+            caches.commands().unregister(commandWatcher);
+            return CommandsForRanges.create(ranges, new TreeMap<>(summaries));
+        }
+    }
+
+    protected void addToQueue(TaskQueue queue)
+    {
+        Invariants.checkState(queue.kind == state || (queue.kind == State.WAITING_TO_LOAD && state == WAITING_TO_SCAN_RANGES), "Invalid queue type: %s vs %s", queue.kind, this, AccordTask::toDescription);
+        Invariants.checkState(this.queued == null, "Already queued with state: %s", this, AccordTask::toDescription);
+        queued = queue;
+        queue.append(this);
+    }
+
+    TaskQueue<?> queued()
+    {
+        return queued;
+    }
+
+    TaskQueue<?> unqueue()
+    {
+        TaskQueue<?> wasQueued = queued;
+        queued.remove(this);
+        queued = null;
+        return wasQueued;
+    }
+
+    TaskQueue<?> unqueueIfQueued()
+    {
+        if (queued == null)
+            return null;
+        return unqueue();
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/accord/CommandsForRanges.java b/src/java/org/apache/cassandra/service/accord/CommandsForRanges.java
index edfde0fb1f..083ebfc205 100644
--- a/src/java/org/apache/cassandra/service/accord/CommandsForRanges.java
+++ b/src/java/org/apache/cassandra/service/accord/CommandsForRanges.java
@@ -60,9 +60,9 @@ public class CommandsForRanges implements CommandsSummary
         this.map = map;
     }
 
-    public static CommandsForRanges create(Ranges ranges, NavigableMap<TxnId, CommandsForRangesLoader.Summary> map)
+    public static CommandsForRanges create(Ranges ranges, NavigableMap<Timestamp, CommandsForRangesLoader.Summary> map)
     {
-        return new CommandsForRanges(ranges, (NavigableMap<Timestamp, CommandsForRangesLoader.Summary>) (NavigableMap<?, ?>) map);
+        return new CommandsForRanges(ranges, map);
     }
 
     @VisibleForTesting
diff --git a/src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java b/src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java
index c338e1116b..4ecfdb5625 100644
--- a/src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java
+++ b/src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java
@@ -19,56 +19,56 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.NavigableMap;
-import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 
 import accord.local.Command;
 import accord.local.KeyHistory;
 import accord.local.RedundantBefore;
+import accord.local.StoreParticipants;
 import accord.primitives.PartialDeps;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
 import accord.primitives.Routable.Domain;
 import accord.primitives.SaveStatus;
 import accord.primitives.Status;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.Routables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
+import accord.utils.Invariants;
 import org.agrona.collections.ObjectHashSet;
-import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.index.accord.RoutesSearcher;
+import org.apache.cassandra.service.accord.AccordCommandStore.Caches;
+import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.utils.Pair;
 
+import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
 
-public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId, Command>
+public class CommandsForRangesLoader implements AccordCache.Listener<TxnId, Command>
 {
+    private final AccordCommandStore commandStore;
     private final RoutesSearcher searcher = new RoutesSearcher();
-    private final AccordCommandStore store;
     private final NavigableMap<TxnId, Ranges> transitive = new TreeMap<>();
     private final ObjectHashSet<TxnId> cachedRangeTxns = new ObjectHashSet<>();
-    // TODO (required): make this configurable, or perhaps backed by READ stage with concurrency limit
 
-    public CommandsForRangesLoader(AccordCommandStore store)
+    public CommandsForRangesLoader(AccordCommandStore commandStore)
     {
-        this.store = store;
-        store.commandCache().register(this);
+        this.commandStore = commandStore;
+        try (ExclusiveCaches caches = commandStore.lockCaches())
+        {
+            caches.commands().register(this);
+        }
     }
 
     @Override
-    public void onAdd(AccordCachingState<TxnId, Command> state)
+    public void onAdd(AccordCacheEntry<TxnId, Command> state)
     {
         TxnId txnId = state.key();
         if (txnId.is(Domain.Range))
@@ -76,256 +76,20 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
     }
 
     @Override
-    public void onEvict(AccordCachingState<TxnId, Command> state)
+    public void onEvict(AccordCacheEntry<TxnId, Command> state)
     {
         TxnId txnId = state.key();
         if (txnId.is(Domain.Range))
             cachedRangeTxns.remove(txnId);
     }
 
-    public AsyncResult<Pair<Watcher, NavigableMap<TxnId, Summary>>> get(@Nullable TxnId primaryTxnId, KeyHistory keyHistory, Ranges ranges)
+    public Loader loader(@Nullable TxnId primaryTxnId, KeyHistory keyHistory, Ranges ranges)
     {
-        RedundantBefore redundantBefore = store.unsafeGetRedundantBefore();
+        RedundantBefore redundantBefore = commandStore.unsafeGetRedundantBefore();
         TxnId minTxnId = redundantBefore.min(ranges, e -> e.gcBefore);
-        Timestamp maxTxnId = primaryTxnId == null || keyHistory == KeyHistory.RECOVERY || !primaryTxnId.is(ExclusiveSyncPoint) ? Timestamp.MAX : primaryTxnId;
-        TxnId findAsDep = primaryTxnId != null && keyHistory == KeyHistory.RECOVERY ? primaryTxnId : null;
-        Watcher watcher = fromCache(findAsDep, ranges, minTxnId, maxTxnId, redundantBefore);
-        ImmutableMap<TxnId, Summary> before = ImmutableMap.copyOf(watcher.get());
-        return AsyncChains.ofCallable(Stage.ACCORD_RANGE_LOADER.executor(), () -> get(ranges, before, findAsDep, minTxnId, maxTxnId, redundantBefore))
-                          .map(map -> Pair.create(watcher, map), store)
-               .beginAsResult();
-    }
-
-    private NavigableMap<TxnId, Summary> get(Ranges ranges, Map<TxnId, Summary> cacheHits, @Nullable TxnId findAsDep, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-    {
-        Set<TxnId> matches = new ObjectHashSet<>();
-        for (Range range : ranges)
-            matches.addAll(intersects(range, minTxnId, maxTxnId));
-        if (matches.isEmpty())
-            return new TreeMap<>();
-        return load(ranges, cacheHits, matches, findAsDep, redundantBefore);
-    }
-
-    private Collection<TxnId> intersects(Range range, TxnId minTxnId, Timestamp maxTxnId)
-    {
-        assert range instanceof TokenRange : "Require TokenRange but given " + range.getClass();
-        Set<TxnId> intersects = searcher.intersects(store.id(), (TokenRange) range, minTxnId, maxTxnId);
-        if (!transitive.isEmpty())
-        {
-            if (intersects.isEmpty())
-                intersects = new ObjectHashSet<>();
-            for (Map.Entry<TxnId, Ranges> e : transitive.tailMap(minTxnId, true).entrySet())
-            {
-                if (e.getValue().intersects(range))
-                    intersects.add(e.getKey());
-            }
-            if (intersects.isEmpty())
-                intersects = Collections.emptySet();
-        }
-        return intersects;
-    }
-
-    public class Watcher implements AccordStateCache.Listener<TxnId, Command>, AutoCloseable
-    {
-        private final Ranges ranges;
-        private final @Nullable TxnId findAsDep;
-        private final TxnId minTxnId;
-        private final Timestamp maxTxnId;
-        private final RedundantBefore redundantBefore;
-
-        private NavigableMap<TxnId, Summary> summaries = null;
-        private Set<AccordCachingState<TxnId, Command>> needToDoubleCheck = null;
-
-        public Watcher(Ranges ranges, @Nullable TxnId findAsDep, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-        {
-            this.ranges = ranges;
-            this.findAsDep = findAsDep;
-            this.minTxnId = minTxnId;
-            this.maxTxnId = maxTxnId;
-            this.redundantBefore = redundantBefore;
-        }
-
-        public NavigableMap<TxnId, Summary> get()
-        {
-            return summaries == null ? Collections.emptyNavigableMap() : summaries;
-        }
-
-        @Override
-        public void onAdd(AccordCachingState<TxnId, Command> n)
-        {
-            if (n.key().domain() != Domain.Range)
-                return;
-
-            if (n.key().compareTo(minTxnId) < 0 || n.key().compareTo(maxTxnId) >= 0)
-                return;
-
-            AccordCachingState.State<TxnId, Command> state = n.state();
-            if (state instanceof AccordCachingState.Loading)
-            {
-                if (needToDoubleCheck == null)
-                    needToDoubleCheck = new ObjectHashSet<>();
-                needToDoubleCheck.add(n);
-                return;
-            }
-            //TODO (required): include FailedToSave?  Most likely need to, but need to improve test coverage to have failed writes
-            if (!(state instanceof AccordCachingState.Loaded
-                  || state instanceof AccordCachingState.Modified
-                  || state instanceof AccordCachingState.Saving))
-                return;
-
-            Command cmd = state.get();
-            if (cmd == null)
-                return;
-            Summary summary = create(cmd, ranges, findAsDep, redundantBefore);
-            if (summary != null)
-            {
-                if (summaries == null)
-                    summaries = new TreeMap<>();
-                summaries.put(summary.txnId, summary);
-            }
-        }
-
-        @Override
-        public void onEvict(AccordCachingState<TxnId, Command> state)
-        {
-            if (needToDoubleCheck == null)
-                return;
-            if (!needToDoubleCheck.remove(state))
-                return;
-            if (state.state() instanceof AccordCachingState.Loading)
-                return; // can't double check
-            onAdd(state);
-        }
-
-        @Override
-        public void close()
-        {
-            store.commandCache().unregister(this);
-            if (needToDoubleCheck != null)
-            {
-                Set<AccordCachingState<TxnId, Command>> copy = needToDoubleCheck;
-                needToDoubleCheck = null;
-                copy.forEach(this::onAdd);
-            }
-            needToDoubleCheck = null;
-        }
-    }
-
-    private Watcher fromCache(@Nullable TxnId findAsDep, Ranges ranges, TxnId minTxnId, Timestamp maxTxnId, RedundantBefore redundantBefore)
-    {
-        Watcher watcher = new Watcher(ranges, findAsDep, minTxnId, maxTxnId, redundantBefore);
-        for (TxnId rangeTxnId : cachedRangeTxns)
-            watcher.onAdd(store.commandCache().getUnsafe(rangeTxnId));
-        store.commandCache().register(watcher);
-        return watcher;
-    }
-
-    private NavigableMap<TxnId, Summary> load(Ranges ranges, Map<TxnId, Summary> cacheHits, Collection<TxnId> possibleTxns, @Nullable TxnId findAsDep, RedundantBefore redundantBefore)
-    {
-        //TODO (required): this logic is kinda duplicate of org.apache.cassandra.service.accord.CommandsForRange.mapReduce
-        // should figure out if this can be improved... also what is correct?
-        NavigableMap<TxnId, Summary> map = new TreeMap<>();
-        for (TxnId txnId : possibleTxns)
-        {
-            if (cacheHits.containsKey(txnId))
-                continue;
-            if (findAsDep == null)
-            {
-                SavedCommand.MinimalCommand cmd = store.loadMinimal(txnId);
-                if (cmd == null)
-                    continue; // unknown command
-                Summary summary = create(cmd, ranges, redundantBefore);
-                if (summary == null)
-                    continue;
-                map.put(txnId, summary);
-
-            }
-            else
-            {
-                Command cmd = store.loadCommand(txnId);
-                if (cmd == null)
-                    continue; // unknown command
-                Summary summary = create(cmd, ranges, findAsDep, redundantBefore);
-                if (summary == null)
-                    continue;
-                map.put(txnId, summary);
-            }
-        }
-        return map;
-    }
-
-    private static Summary create(Command cmd, Ranges cacheRanges, @Nullable TxnId findAsDep, @Nullable RedundantBefore redundantBefore)
-    {
-        //TODO (required, correctness): C* did Invalidated, accord-core did Erased... what is correct?
-        SaveStatus saveStatus = cmd.saveStatus();
-        if (saveStatus == SaveStatus.Invalidated
-            || saveStatus == SaveStatus.Erased
-            || !saveStatus.hasBeen(Status.PreAccepted))
-            return null;
-        if (cmd.partialTxn() == null)
-            return null;
-
-        Ranges keysOrRanges = cmd.participants().touches().toRanges();
-        if (keysOrRanges.domain() != Domain.Range)
-            throw new AssertionError(String.format("Txn keys are not range for %s", cmd.partialTxn()));
-        Ranges ranges = (Ranges) keysOrRanges;
-
-        ranges = ranges.slice(cacheRanges, Routables.Slice.Minimal);
-        if (ranges.isEmpty())
-            return null;
-
-        if (redundantBefore != null)
-        {
-            Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
-                if (e.gcBefore.compareTo(cmd.txnId()) < 0)
-                    return accum;
-                return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
-            }, ranges, ignore -> false);
-
-            if (newRanges.isEmpty())
-                return null;
-        }
-
-        PartialDeps partialDeps = cmd.partialDeps();
-        boolean hasAsDep = findAsDep != null && partialDeps != null && partialDeps.rangeDeps.intersects(findAsDep, ranges);
-        return new Summary(cmd.txnId(), cmd.executeAt(), saveStatus, ranges, findAsDep, hasAsDep);
-    }
-
-    private static Summary create(SavedCommand.MinimalCommand cmd, Ranges cacheRanges, @Nullable RedundantBefore redundantBefore)
-    {
-        //TODO (required, correctness): C* did Invalidated, accord-core did Erased... what is correct?
-        SaveStatus saveStatus = cmd.saveStatus;
-        if (saveStatus == null
-            || saveStatus == SaveStatus.Invalidated
-            || saveStatus == SaveStatus.Erased
-            || !saveStatus.hasBeen(Status.PreAccepted))
-            return null;
-
-        if (cmd.participants == null)
-            return null;
-
-        Ranges keysOrRanges = cmd.participants.touches().toRanges();
-        if (keysOrRanges.domain() != Domain.Range)
-            throw new AssertionError(String.format("Txn keys are not range for %s", cmd.participants));
-        Ranges ranges = (Ranges) keysOrRanges;
-
-        ranges = ranges.slice(cacheRanges, Routables.Slice.Minimal);
-        if (ranges.isEmpty())
-            return null;
-
-        if (redundantBefore != null)
-        {
-            Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
-                if (e.gcBefore.compareTo(cmd.txnId) < 0)
-                    return accum;
-                return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
-            }, ranges, ignore -> false);
-
-            if (newRanges.isEmpty())
-                return null;
-        }
-
-        return new Summary(cmd.txnId, cmd.executeAt, saveStatus, ranges, null, false);
+        Timestamp maxTxnId = primaryTxnId == null || keyHistory == KeyHistory.RECOVER || !primaryTxnId.is(ExclusiveSyncPoint) ? Timestamp.MAX : primaryTxnId;
+        TxnId findAsDep = primaryTxnId != null && keyHistory == KeyHistory.RECOVER ? primaryTxnId : null;
+        return new Loader(ranges, redundantBefore, minTxnId, maxTxnId, findAsDep);
     }
 
     public void mergeTransitive(TxnId txnId, Ranges ranges, BiFunction<? super Ranges, ? super Ranges, ? extends Ranges> remappingFunction)
@@ -370,7 +134,7 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
 
         public Summary slice(Ranges slice)
         {
-            return new Summary(txnId, executeAt, saveStatus, ranges.slice(slice, Routables.Slice.Minimal), findAsDep, hasAsDep);
+            return new Summary(txnId, executeAt, saveStatus, ranges == null ? null : ranges.slice(slice, Minimal), findAsDep, hasAsDep);
         }
 
         @Override
@@ -386,4 +150,150 @@ public class CommandsForRangesLoader implements AccordStateCache.Listener<TxnId,
                    '}';
         }
     }
+
+    public class Loader
+    {
+        final Ranges searchRanges;
+        final RedundantBefore redundantBefore;
+        final TxnId minTxnId;
+        final Timestamp maxTxnId;
+        @Nullable final TxnId findAsDep;
+
+        public Loader(Ranges searchRanges, RedundantBefore redundantBefore, TxnId minTxnId, Timestamp maxTxnId, @Nullable TxnId findAsDep)
+        {
+            this.searchRanges = searchRanges;
+            this.redundantBefore = redundantBefore;
+            this.minTxnId = minTxnId;
+            this.maxTxnId = maxTxnId;
+            this.findAsDep = findAsDep;
+        }
+
+        public Collection<TxnId> intersects()
+        {
+            ObjectHashSet<TxnId> txnIds = new ObjectHashSet<>();
+            for (Range range : searchRanges)
+            {
+                searcher.intersects(commandStore.id(), (TokenRange) range, minTxnId, maxTxnId, txnIds::add);
+            }
+            if (!transitive.isEmpty())
+            {
+                for (Map.Entry<TxnId, Ranges> e : transitive.tailMap(minTxnId, true).entrySet())
+                {
+                    if (e.getValue().intersects(searchRanges))
+                        txnIds.add(e.getKey());
+                }
+            }
+            return txnIds;
+        }
+
+        public void forEachInCache(Consumer<Summary> forEach, Caches caches)
+        {
+            for (TxnId txnId : cachedRangeTxns)
+            {
+                AccordCacheEntry<TxnId, Command> state = caches.commands().getUnsafe(txnId);
+                Summary summary = from(state);
+                if (summary != null)
+                    forEach.accept(summary);
+            }
+        }
+
+        public Summary load(TxnId txnId)
+        {
+            if (findAsDep == null)
+            {
+                SavedCommand.MinimalCommand cmd = commandStore.loadMinimal(txnId);
+                return cmd == null ? null : from(cmd);
+            }
+            else
+            {
+                Command cmd = commandStore.loadCommand(txnId);
+                return cmd == null ? null : from(cmd);
+            }
+        }
+
+        public Summary from(AccordCacheEntry<TxnId, Command> state)
+        {
+            if (state.key().domain() != Domain.Range)
+                return null;
+
+            switch (state.status())
+            {
+                default: throw new AssertionError("Unhandled status: " + state.status());
+                case LOADING:
+                case WAITING_TO_LOAD:
+                case UNINITIALIZED:
+                    return null;
+
+                case LOADED:
+                case MODIFIED:
+                case SAVING:
+                case FAILED_TO_SAVE:
+            }
+
+            TxnId txnId = state.key();
+            if (!txnId.isVisible() || txnId.compareTo(minTxnId) < 0 || txnId.compareTo(maxTxnId) >= 0)
+                return null;
+
+            Command command = state.getExclusive();
+            if (command == null)
+                return null;
+            return from(command);
+        }
+
+        public Summary from(Command cmd)
+        {
+            return from(cmd.txnId(), cmd.executeAt(), cmd.saveStatus(), cmd.participants(), cmd.partialDeps());
+        }
+
+        public Summary from(SavedCommand.MinimalCommand cmd)
+        {
+            Invariants.checkState(findAsDep == null);
+            return from(cmd.txnId, cmd.executeAt, cmd.saveStatus, cmd.participants, null);
+        }
+
+        private Summary from(TxnId txnId, Timestamp executeAt, SaveStatus saveStatus, StoreParticipants participants, @Nullable PartialDeps partialDeps)
+        {
+            if (saveStatus == SaveStatus.Invalidated
+                || saveStatus == SaveStatus.Erased
+                || !saveStatus.hasBeen(Status.PreAccepted))
+                return null;
+
+            if (participants == null)
+                return null;
+
+            Ranges keysOrRanges = participants.touches().toRanges();
+            if (keysOrRanges.domain() != Domain.Range)
+                throw new AssertionError(String.format("Txn keys are not range for %s", participants));
+            Ranges ranges = keysOrRanges;
+
+            ranges = ranges.slice(searchRanges, Minimal);
+            if (ranges.isEmpty())
+                return null;
+
+            if (redundantBefore != null)
+            {
+                Ranges newRanges = redundantBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
+                    if (e.gcBefore.compareTo(txnId) < 0)
+                        return accum;
+                    return accum.without(Ranges.of(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end)));
+                }, ranges, ignore -> false);
+
+                if (newRanges.isEmpty())
+                    return null;
+
+                ranges = newRanges;
+            }
+
+            Invariants.checkState(partialDeps != null || findAsDep == null || !saveStatus.known.deps.hasProposedOrDecidedDeps());
+            boolean hasAsDep = false;
+            if (partialDeps != null)
+            {
+                Ranges depRanges = partialDeps.rangeDeps.ranges(txnId);
+                if (depRanges != null && depRanges.containsAll(ranges))
+                    hasAsDep = true;
+            }
+
+            return new Summary(txnId, executeAt, saveStatus, ranges, findAsDep, hasAsDep);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java
index 4d3505f626..f78eb3c95c 100644
--- a/src/java/org/apache/cassandra/service/accord/IAccordService.java
+++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java
@@ -51,6 +51,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordScheduler;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.tcm.Epoch;
@@ -114,6 +115,7 @@ public interface IAccordService
     long currentEpoch();
 
     void setCacheSize(long kb);
+    void setWorkingSetSize(long kb);
 
     TopologyManager topology();
 
@@ -152,6 +154,8 @@ public interface IAccordService
      */
     CompactionInfo getCompactionInfo();
 
+    AccordAgent agent();
+
     default Id nodeId() { throw new UnsupportedOperationException(); }
 
     List<CommandStoreTxnBlockedGraph> debugTxnBlockedGraph(TxnId txnId);
diff --git a/src/java/org/apache/cassandra/service/accord/IJournal.java b/src/java/org/apache/cassandra/service/accord/IJournal.java
index 849173c9b5..61ba6a5ad7 100644
--- a/src/java/org/apache/cassandra/service/accord/IJournal.java
+++ b/src/java/org/apache/cassandra/service/accord/IJournal.java
@@ -39,7 +39,7 @@ public interface IJournal
     NavigableMap<Timestamp, Ranges> loadSafeToRead(int commandStoreId);
     CommandStores.RangesForEpoch.Snapshot loadRangesForEpoch(int commandStoreId);
 
-    void appendCommand(int store, SavedCommand.DiffWriter value, Runnable onFlush);
+    void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush);
     Persister<DurableBefore, DurableBefore> durableBeforePersister();
     void persistStoreState(int store,
                            // TODO: this class should not live under ASCS
diff --git a/src/java/org/apache/cassandra/service/accord/JournalKey.java b/src/java/org/apache/cassandra/service/accord/JournalKey.java
index 4292387ee7..d8765ddf1f 100644
--- a/src/java/org/apache/cassandra/service/accord/JournalKey.java
+++ b/src/java/org/apache/cassandra/service/accord/JournalKey.java
@@ -92,6 +92,14 @@ public final class JournalKey
             serializeTxnId(key.id, out);
         }
 
+        @Override
+        public void serialize(JournalKey key, ByteBuffer out, int userVersion) throws IOException
+        {
+            out.putInt(key.commandStoreId);
+            out.put((byte) key.type.id);
+            serializeTxnId(key.id, out);
+        }
+
         private void serialize(JournalKey key, byte[] out)
         {
             ByteArrayUtil.putInt(out, CS_ID_OFFSET, key.commandStoreId);
@@ -117,6 +125,15 @@ public final class JournalKey
             return new JournalKey(txnId, Type.fromId(type), commandStoreId);
         }
 
+        @Override
+        public JournalKey deserialize(ByteBuffer buffer, int userVersion)
+        {
+            int commandStoreId = buffer.getInt();
+            int type = buffer.get();
+            TxnId txnId = deserializeTxnId(buffer);
+            return new JournalKey(txnId, Type.fromId(type), commandStoreId);
+        }
+
         private void serializeTxnId(TxnId txnId, DataOutputPlus out) throws IOException
         {
             out.writeLong(txnId.msb);
@@ -132,6 +149,14 @@ public final class JournalKey
             return TxnId.fromBits(msb, lsb, new Id(nodeId));
         }
 
+        private TxnId deserializeTxnId(ByteBuffer in)
+        {
+            long msb = in.getLong();
+            long lsb = in.getLong();
+            int nodeId = in.getInt();
+            return TxnId.fromBits(msb, lsb, new Id(nodeId));
+        }
+
         private void serializeTxnId(TxnId txnId, byte[] out)
         {
             ByteArrayUtil.putLong(out, MSB_OFFSET, txnId.msb);
@@ -139,6 +164,13 @@ public final class JournalKey
             ByteArrayUtil.putInt(out, NODE_OFFSET, txnId.node.id);
         }
 
+        private void serializeTxnId(TxnId txnId, ByteBuffer out)
+        {
+            out.putLong(txnId.msb);
+            out.putLong(txnId.lsb);
+            out.putInt(txnId.node.id);
+        }
+
         private TxnId deserializeTxnId(ByteBuffer buffer, int position)
         {
             long msb = buffer.getLong(position + MSB_OFFSET);
diff --git a/src/java/org/apache/cassandra/service/accord/SavedCommand.java b/src/java/org/apache/cassandra/service/accord/SavedCommand.java
index 0bd21ef056..f4ff9ee399 100644
--- a/src/java/org/apache/cassandra/service/accord/SavedCommand.java
+++ b/src/java/org/apache/cassandra/service/accord/SavedCommand.java
@@ -20,11 +20,13 @@ package org.apache.cassandra.service.accord;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.EnumSet;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import accord.api.Agent;
 import accord.api.Result;
 import accord.local.Cleanup;
 import accord.local.Command;
@@ -89,31 +91,24 @@ public class SavedCommand
     }
 
     // TODO: maybe rename this and enclosing classes?
-    public static class DiffWriter implements Journal.Writer
+    public static class Writer implements Journal.Writer
     {
-        private final Command before;
         private final Command after;
         private final TxnId txnId;
+        private final int flags;
 
-        // TODO: improve encapsulationd
         @VisibleForTesting
-        public DiffWriter(Command before, Command after)
+        public Writer(Command after, int flags)
         {
-            this(after.txnId(), before, after);
+            this(after.txnId(), after, flags);
         }
 
         @VisibleForTesting
-        public DiffWriter(TxnId txnId, Command before, Command after)
+        public Writer(TxnId txnId, Command after, int flags)
         {
             this.txnId = txnId;
-            this.before = before;
             this.after = after;
-        }
-
-        @VisibleForTesting
-        public Command before()
-        {
-            return before;
+            this.flags = flags;
         }
 
         @VisibleForTesting
@@ -124,7 +119,7 @@ public class SavedCommand
 
         public void write(DataOutputPlus out, int userVersion) throws IOException
         {
-            serialize(before, after, out, userVersion);
+            serialize(after, flags, out, userVersion);
         }
 
         public TxnId key()
@@ -133,30 +128,37 @@ public class SavedCommand
         }
     }
 
-    public static ByteBuffer asSerializedDiff(Command after, int userVersion) throws IOException
+    public static @Nullable ByteBuffer asSerializedDiff(Command before, Command after, int userVersion) throws IOException
     {
         try (DataOutputBuffer out = new DataOutputBuffer())
         {
-            diff(null, after).write(out, userVersion);
+            Writer writer = diff(before, after);
+            if (writer == null)
+                return null;
+
+            writer.write(out, userVersion);
             return out.asNewBuffer();
         }
     }
 
     @Nullable
-    public static DiffWriter diff(Command original, Command current)
+    public static Writer diff(Command original, Command current)
     {
         if (original == current
             || current == null
-            || current.saveStatus() == SaveStatus.Uninitialised
-            || !anyFieldChanged(original, current))
+            || current.saveStatus() == SaveStatus.Uninitialised)
             return null;
-        return new SavedCommand.DiffWriter(original, current);
+
+        int flags = validateFlags(getFlags(original, current));
+        if (!anyFieldChanged(flags))
+            return null;
+
+        return new Writer(current, flags);
     }
 
     // TODO (required): calculate flags once
-    private static boolean anyFieldChanged(Command before, Command after)
+    private static boolean anyFieldChanged(int flags)
     {
-        int flags = validateFlags(getFlags(before, after));
         return (flags >>> 16) != 0;
     }
 
@@ -166,9 +168,9 @@ public class SavedCommand
         return flags;
     }
     
-    public static void serialize(Command before, Command after, DataOutputPlus out, int userVersion) throws IOException
+    public static void serialize(Command after, int flags, DataOutputPlus out, int userVersion) throws IOException
     {
-        int flags = validateFlags(getFlags(before, after));
+        Invariants.checkState(flags != 0);
         out.writeInt(flags);
 
         int iterable = toIterableSetFields(flags);
@@ -230,7 +232,7 @@ public class SavedCommand
     }
 
     @VisibleForTesting
-    static int getFlags(Command before, Command after)
+    public static int getFlags(Command before, Command after)
     {
         int flags = 0;
 
@@ -296,6 +298,17 @@ public class SavedCommand
         return (oldFlags & (0x10000 << field.ordinal())) != 0;
     }
 
+    static EnumSet<Fields> getFieldsChanged(int flags)
+    {
+        EnumSet<Fields> fields = EnumSet.noneOf(Fields.class);
+        for (Fields field : Fields.FIELDS)
+        {
+            if ((flags & (0x10000 << field.ordinal())) != 0)
+                fields.add(field);
+        }
+        return fields;
+    }
+
     static int toIterableSetFields(int flags)
     {
         return flags >>> 16;
@@ -536,7 +549,7 @@ public class SavedCommand
             return count;
         }
 
-        public Cleanup shouldCleanup(RedundantBefore redundantBefore, DurableBefore durableBefore)
+        public Cleanup shouldCleanup(Agent agent, RedundantBefore redundantBefore, DurableBefore durableBefore)
         {
             if (!nextCalled)
                 return NO;
@@ -544,7 +557,7 @@ public class SavedCommand
             if (saveStatus == null || participants == null)
                 return Cleanup.NO;
 
-            Cleanup cleanup = Cleanup.shouldCleanupPartial(txnId, saveStatus, durability, participants, redundantBefore, durableBefore);
+            Cleanup cleanup = Cleanup.shouldCleanupPartial(agent, txnId, saveStatus, durability, participants, redundantBefore, durableBefore);
             if (this.cleanup != null && this.cleanup.compareTo(cleanup) > 0)
                 cleanup = this.cleanup;
             return cleanup;
@@ -661,6 +674,7 @@ public class SavedCommand
         public void serialize(DataOutputPlus out, int userVersion) throws IOException
         {
             Invariants.checkState(mask == 0);
+            Invariants.checkState(flags != 0);
             out.writeInt(validateFlags(flags));
 
             int iterable = toIterableSetFields(flags);
@@ -718,12 +732,11 @@ public class SavedCommand
             }
         }
 
-        // TODO: we seem to be writing some form of empty transaction
-        @SuppressWarnings({ "rawtypes", "unchecked" })
         public void deserializeNext(DataInputPlus in, int userVersion) throws IOException
         {
             Invariants.checkState(txnId != null);
-            final int flags = in.readInt();
+            int flags = in.readInt();
+            Invariants.checkState(flags != 0);
             nextCalled = true;
             count++;
 
diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
index d1af6f803c..70ea4e8366 100644
--- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
+++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
@@ -57,6 +57,7 @@ import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static accord.primitives.Routable.Domain.Key;
+import static accord.utils.Invariants.illegalState;
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -185,16 +186,10 @@ public class AccordAgent implements Agent
         return AccordMetrics.Listener.instance;
     }
 
-    @Override
-    public long replyTimeout(ReplyContext replyContext, TimeUnit units)
-    {
-        return Math.max(1, units.convert(((ResponseContext)replyContext).expiresAtNanos() - Clock.Global.nanoTime(), NANOSECONDS));
-    }
-
     @Override
     public long attemptCoordinationDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int retryCount)
     {
-        SafeCommand safeCommand = safeStore.ifInitialised(txnId);
+        SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId);
         Invariants.nonNull(safeCommand);
 
         Command command = safeCommand.current();
@@ -250,4 +245,17 @@ public class AccordAgent implements Agent
         // TODO (expected): integrate with contention backoff
         return units.convert((1L << Math.min(retryCount, 4)), SECONDS);
     }
+
+    @Override
+    public long expiresAt(ReplyContext replyContext, TimeUnit unit)
+    {
+        return unit.convert(((ResponseContext)replyContext).expiresAtNanos(), NANOSECONDS);
+    }
+
+    @Override
+    public void onViolation(String message)
+    {
+        try { throw illegalState(message); }
+        catch (Throwable t) { logger.error("Consistency violation", t); }
+    }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java b/src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java
index 18d6926bc5..bf0d5d287a 100644
--- a/src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java
+++ b/src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java
@@ -19,7 +19,6 @@
 package org.apache.cassandra.service.accord.api;
 
 import java.io.IOException;
-import java.util.Objects;
 
 import accord.primitives.RoutableKey;
 import org.apache.cassandra.dht.Token;
@@ -65,7 +64,7 @@ public abstract class AccordRoutableKey implements RoutableKey
     @Override
     public int hashCode()
     {
-        return Objects.hash(table, token().tokenHash());
+        return table.hashCode() * 31 + token().tokenHash();
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java b/src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java
index 68ab848885..ac9f5329fd 100644
--- a/src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java
+++ b/src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java
@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.TreeMap;
 
 import accord.api.Key;
@@ -112,7 +111,7 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         @Override
         public int hashCode()
         {
-            return Objects.hash(table, isMin);
+            return table.hashCode() * (isMin ? 31 : 1);
         }
 
         @Override
@@ -462,5 +461,11 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         {
             return subSplitter.splitRange(range, from, to, numSplits);
         }
+
+        @Override
+        public int numberOfSplitsPossible(Range range)
+        {
+            return subSplitter.numberOfSplitsPossible(range);
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java b/src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java
index dec0cbb225..b8e0b8755f 100644
--- a/src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java
+++ b/src/java/org/apache/cassandra/service/accord/api/AccordScheduler.java
@@ -30,7 +30,7 @@ import org.apache.cassandra.concurrent.Shutdownable;
 
 public class AccordScheduler implements Scheduler, Shutdownable
 {
-    private final ScheduledExecutorPlus scheduledExecutor = ExecutorFactory.Global.executorFactory().scheduled("AccordScheduled");
+    private final ScheduledExecutorPlus scheduledExecutor = ExecutorFactory.Global.executorFactory().scheduled(false, "AccordScheduled");
 
     private static class ScheduledFutureWrapper implements Scheduled
     {
diff --git a/src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java b/src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java
deleted file mode 100644
index 7f6b30fb7c..0000000000
--- a/src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord.async;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.api.RoutingKey;
-import accord.local.KeyHistory;
-import accord.local.PreLoadContext;
-import accord.local.cfk.CommandsForKey;
-import accord.primitives.AbstractKeys;
-import accord.primitives.AbstractRanges;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import accord.utils.async.Observable;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.service.accord.AccordCachingState;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForRanges;
-import org.apache.cassandra.service.accord.AccordSafeState;
-import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.CommandsForRangesLoader;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.utils.NoSpamLogger;
-import org.apache.cassandra.utils.Pair;
-
-public class AsyncLoader
-{
-    private static final Logger logger = LoggerFactory.getLogger(AsyncLoader.class);
-    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.MINUTES);
-
-    enum State
-    {
-        INITIALIZED,
-        SETUP,
-        LOADING,
-        FINISHED
-    }
-
-    private State state = State.INITIALIZED;
-    private final AccordCommandStore commandStore;
-
-    private final Iterable<TxnId> txnIds;
-    private final Unseekables<?> keysOrRanges;
-    private final KeyHistory keyHistory;
-
-    protected AsyncResult<?> readResult;
-
-    public AsyncLoader(AccordCommandStore commandStore, Iterable<TxnId> txnIds, Unseekables<?> keysOrRanges, KeyHistory keyHistory)
-    {
-        this.commandStore = commandStore;
-        this.txnIds = txnIds;
-        this.keysOrRanges = keysOrRanges;
-        this.keyHistory = keyHistory;
-    }
-
-    protected static Iterable<TxnId> txnIds(PreLoadContext context)
-    {
-        TxnId primaryid = context.primaryTxnId();
-        Collection<TxnId> additionalIds = context.additionalTxnIds();
-        if (primaryid == null) return additionalIds;
-        if (additionalIds.isEmpty()) return Collections.singleton(primaryid);
-        return Iterables.concat(Collections.singleton(primaryid), additionalIds);
-    }
-
-    private static <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReadsForKey(K key,
-                                                                                                Map<K, S> context,
-                                                                                                AccordStateCache.Instance<K, V, S> cache,
-                                                                                                List<AsyncChain<?>> listenChains)
-    {
-        referenceAndAssembleReadsForKey(key, context, cache, listenChains, null);
-    }
-
-    private static <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReadsForKey(K key,
-                                                                                                Map<K, S> context,
-                                                                                                AccordStateCache.Instance<K, V, S> cache,
-                                                                                                List<AsyncChain<?>> listenChains,
-                                                                                                @Nullable ExecutorPlus loadExecutor)
-    {
-        S safeRef = cache.acquire(key, loadExecutor);
-        if (context.putIfAbsent(key, safeRef) != null)
-        {
-            noSpamLogger.warn("Context {} contained key {} more than once", context, key);
-            cache.release(safeRef);
-            return;
-        }
-        AccordCachingState.Status status = safeRef.globalStatus(); // globalStatus() completes
-        switch (status)
-        {
-            default: throw new IllegalStateException("Unhandled global state: " + status);
-            case LOADING:
-                listenChains.add(safeRef.loading());
-                break;
-            case SAVING:
-                // make sure we work with a completed state that supports get() and set()
-                listenChains.add(safeRef.saving());
-                break;
-            case LOADED:
-            case MODIFIED:
-            case FAILED_TO_SAVE:
-                break;
-            case FAILED_TO_LOAD:
-                // TODO (required): if this triggers, we trigger some other illegal state in cache management
-                throw new RuntimeException(safeRef.failure());
-        }
-    }
-
-    private void referenceAndAssembleReadsForKey(RoutingKey key,
-                                                 AsyncOperation.Context context,
-                                                 List<AsyncChain<?>> listenChains)
-    {
-        referenceAndAssembleReadsForKey(key, context, listenChains, null);
-    }
-
-    private void referenceAndAssembleReadsForKey(RoutingKey key,
-                                                 AsyncOperation.Context context,
-                                                 List<AsyncChain<?>> listenChains,
-                                                 @Nullable ExecutorPlus loadExecutor)
-    {
-        // recovery operations also need the deps data for their preaccept logic
-        switch (keyHistory)
-        {
-            case TIMESTAMPS:
-                referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains, loadExecutor);
-                break;
-            case COMMANDS:
-            case RECOVERY:
-                referenceAndAssembleReadsForKey(key, context.commandsForKey, commandStore.commandsForKeyCache(), listenChains, loadExecutor);
-            case NONE:
-                break;
-            default: throw new IllegalArgumentException("Unhandled keyhistory: " + keyHistory);
-        }
-    }
-
-    private <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReads(Iterable<? extends K> keys,
-                                                                                   Map<K, S> context,
-                                                                                   AccordStateCache.Instance<K, V, S> cache,
-                                                                                   List<AsyncChain<?>> listenChains)
-    {
-        keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, cache, listenChains));
-    }
-
-    private AsyncResult<?> referenceAndDispatchReads(@Nullable TxnId primaryTxnId, AsyncOperation.Context context)
-    {
-        List<AsyncChain<?>> chains = new ArrayList<>();
-
-        referenceAndAssembleReads(txnIds, context.commands, commandStore.commandCache(), chains);
-
-        switch (keysOrRanges.domain())
-        {
-            case Key:
-                AbstractKeys<RoutingKey> keys = (AbstractKeys<RoutingKey>) keysOrRanges;
-                keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
-                break;
-            case Range:
-                chains.add(referenceAndDispatchReadsForRange(primaryTxnId, context));
-                break;
-            default:
-                throw new UnsupportedOperationException("Unable to process keys of " + keysOrRanges.domain());
-        }
-
-        return !chains.isEmpty() ? AsyncChains.reduce(chains, (a, b) -> null).beginAsResult() : null;
-    }
-
-    private AsyncChain<?> referenceAndDispatchReadsForRange(@Nullable TxnId primaryTxnId, AsyncOperation.Context context)
-    {
-        if (keyHistory == KeyHistory.NONE)
-            return AsyncChains.success(null);
-
-        Ranges ranges = ((AbstractRanges) keysOrRanges).toRanges();
-
-        List<AsyncChain<?>> root = new ArrayList<>(ranges.size() + 1);
-        class Watcher implements AccordStateCache.Listener<RoutingKey, CommandsForKey>
-        {
-            // TODO (required): streams prohibited in hot path
-            private final Set<TokenKey> cached = commandStore.commandsForKeyCache().stream()
-                                                                 .map(n -> (TokenKey) n.key())
-                                                                 .filter(ranges::contains)
-                                                                 .collect(Collectors.toSet());
-
-            @Override
-            public void onAdd(AccordCachingState<RoutingKey, CommandsForKey> state)
-            {
-                TokenKey pk = (TokenKey) state.key();
-                if (ranges.contains(pk))
-                    cached.add(pk);
-            }
-        }
-
-        // TODO (required): this needs to be optimised (e.g. to not load redundant commands, but maybe to be avoided altogether with async evaluation)
-        Watcher watcher = new Watcher();
-        commandStore.commandsForKeyCache().register(watcher);
-        root.add(findOverlappingKeys(ranges).flatMap(keys -> {
-            commandStore.commandsForKeyCache().unregister(watcher);
-            if (keys.isEmpty() && watcher.cached.isEmpty())
-                return AsyncChains.success(null);
-            Set<? extends RoutingKey> set = ImmutableSet.<RoutingKey>builder().addAll(watcher.cached).addAll(keys).build();
-            List<AsyncChain<?>> chains = new ArrayList<>();
-            set.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains, Stage.ACCORD_RANGE_LOADER.executor()));
-            return chains.isEmpty() ? AsyncChains.success(null) : AsyncChains.reduce(chains, (a, b) -> null);
-        }, commandStore));
-
-        AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain = commandStore.diskCommandsForRanges().get(primaryTxnId, keyHistory, ranges);
-        root.add(chain);
-        context.commandsForRanges = new AccordSafeCommandsForRanges(ranges, chain);
-
-        return AsyncChains.all(root);
-    }
-
-    private AsyncChain<List<? extends RoutingKey>> findOverlappingKeys(Ranges ranges)
-    {
-        if (ranges.isEmpty())
-        {
-            // During topology changes some shards may be included with empty ranges
-            return AsyncChains.success(Collections.emptyList());
-        }
-
-        List<AsyncChain<List<TokenKey>>> chains = new ArrayList<>(ranges.size());
-        for (Range range : ranges)
-            chains.add(findOverlappingKeys(range));
-        return AsyncChains.reduce(chains, (a, b) -> ImmutableList.<RoutingKey>builderWithExpectedSize(a.size() + b.size()).addAll(a).addAll(b).build());
-    }
-
-    private AsyncChain<List<TokenKey>> findOverlappingKeys(Range range)
-    {
-        // save to a variable as java gets confused when `.map` is called on the result of asChain
-        AsyncChain<List<TokenKey>> map = Observable.asChain(callback ->
-                                                               AccordKeyspace.findAllKeysBetween(commandStore.id(),
-                                                                                                 (AccordRoutingKey) range.start(), range.startInclusive(),
-                                                                                                 (AccordRoutingKey) range.end(), range.endInclusive(),
-                                                                                                 callback),
-                                                               Collectors.toList());
-        return map.map(ImmutableList::copyOf);
-    }
-
-    @VisibleForTesting
-    void state(State state)
-    {
-        this.state = state;
-    }
-
-    public boolean load(@Nullable TxnId primaryTxnId, AsyncOperation.Context context, BiConsumer<Object, Throwable> callback)
-    {
-        logger.trace("Running load for {} with state {}: {} {}", callback, state, txnIds, keysOrRanges);
-        commandStore.checkInStoreThread();
-        switch (state)
-        {
-            case INITIALIZED:
-                state(State.SETUP);
-
-            case SETUP:
-                readResult = referenceAndDispatchReads(primaryTxnId, context);
-                state(State.LOADING);
-
-            case LOADING:
-                if (readResult != null)
-                {
-                    if (readResult.isSuccess())
-                    {
-                        logger.trace("Read result succeeded for {}", callback);
-                        readResult = null;
-                    }
-                    else
-                    {
-                        logger.trace("Adding callback for read result: {}", callback);
-                        readResult.addCallback(callback, commandStore.executor());
-                        break;
-                    }
-                }
-                state(State.FINISHED);
-
-            case FINISHED:
-                break;
-
-            default:
-                throw new IllegalStateException("Unexpected state: " + state);
-        }
-
-        if (logger.isTraceEnabled())
-            logger.trace("Exiting load for {} with state {}: {} {}", callback, state, txnIds, keysOrRanges);
-
-        return state == State.FINISHED;
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java b/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java
deleted file mode 100644
index 51dbb0a6c2..0000000000
--- a/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord.async;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import accord.api.RoutingKey;
-import accord.local.Command;
-import accord.local.CommandStore;
-import accord.local.PreLoadContext;
-import accord.local.SafeCommandStore;
-import accord.primitives.TxnId;
-import accord.primitives.Unseekables;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.Cancellable;
-import org.agrona.collections.Object2ObjectHashMap;
-import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeCommandStore;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForRanges;
-import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
-import org.apache.cassandra.service.accord.SavedCommand;
-import org.apache.cassandra.utils.concurrent.Condition;
-
-import static org.apache.cassandra.service.accord.async.AsyncLoader.txnIds;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.COMPLETING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FAILED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FINISHED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.INITIALIZED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.LOADING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.PREPARING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.RUNNING;
-
-public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements Runnable, Function<SafeCommandStore, R>, Cancellable
-{
-    private static final Logger logger = LoggerFactory.getLogger(AsyncOperation.class);
-
-    private static class LoggingProps
-    {
-        private static final String COMMAND_STORE = "command_store";
-        private static final String ASYNC_OPERATION = "async_op";
-    }
-
-    static class Context
-    {
-        final Object2ObjectHashMap<TxnId, AccordSafeCommand> commands = new Object2ObjectHashMap<>();
-        final Object2ObjectHashMap<RoutingKey, AccordSafeTimestampsForKey> timestampsForKey = new Object2ObjectHashMap<>();
-        final Object2ObjectHashMap<RoutingKey, AccordSafeCommandsForKey> commandsForKey = new Object2ObjectHashMap<>();
-        @Nullable
-        AccordSafeCommandsForRanges commandsForRanges = null;
-
-        void releaseResources(AccordCommandStore commandStore)
-        {
-            // TODO (expected): we should destructively iterate to avoid invoking second time in fail; or else read and set to null
-            commands.forEach((k, v) -> commandStore.commandCache().release(v));
-            commands.clear();
-            timestampsForKey.forEach((k, v) -> commandStore.timestampsForKeyCache().release(v));
-            timestampsForKey.clear();
-            commandsForKey.forEach((k, v) -> commandStore.commandsForKeyCache().release(v));
-            commandsForKey.clear();
-        }
-
-        void revertChanges()
-        {
-            commands.forEach((k, v) -> v.revert());
-            timestampsForKey.forEach((k, v) -> v.revert());
-            commandsForKey.forEach((k, v) -> v.revert());
-            if (commandsForRanges != null)
-                commandsForRanges.revert();
-        }
-    }
-
-    enum State
-    {
-        INITIALIZED, LOADING, PREPARING, RUNNING, COMPLETING, AWAITING_FLUSH, FINISHED, FAILED;
-
-        boolean isComplete()
-        {
-            return this == FINISHED || this == FAILED;
-        }
-    }
-
-    private State state = INITIALIZED;
-    private final AccordCommandStore commandStore;
-    private final PreLoadContext preLoadContext;
-    private final Context context = new Context();
-    private AccordSafeCommandStore safeStore;
-    private final AsyncLoader loader;
-    private R result;
-    private final String loggingId;
-    private BiConsumer<? super R, Throwable> callback;
-
-    private List<Command> sanityCheck = null;
-
-    private void setLoggingIds()
-    {
-        MDC.put(LoggingProps.COMMAND_STORE, commandStore.loggingId);
-        MDC.put(LoggingProps.ASYNC_OPERATION, loggingId);
-    }
-
-    private void clearLoggingIds()
-    {
-        MDC.remove(LoggingProps.COMMAND_STORE);
-        MDC.remove(LoggingProps.ASYNC_OPERATION);
-    }
-
-    public AsyncOperation(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-    {
-        this.loggingId = "0x" + Integer.toHexString(System.identityHashCode(this));
-        this.commandStore = commandStore;
-        this.preLoadContext = preLoadContext;
-        this.loader = createAsyncLoader(commandStore, preLoadContext);
-
-        if (logger.isTraceEnabled())
-        {
-            setLoggingIds();
-            logger.trace("Created {} on {}", this, commandStore);
-            clearLoggingIds();
-        }
-    }
-
-    @Override
-    public String toString()
-    {
-        return "AsyncOperation{" + state + "}-" + loggingId;
-    }
-
-    AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-    {
-        return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys(), preLoadContext.keyHistory());
-    }
-
-    private void onLoaded(Object o, Throwable throwable)
-    {
-        if (throwable != null)
-        {
-            logger.error(String.format("Operation %s failed", this), throwable);
-            fail(throwable);
-        }
-        else
-        {
-            run();
-        }
-    }
-
-    private void state(State state)
-    {
-        this.state = state;
-    }
-
-    private void finish(R result, Throwable failure)
-    {
-        try
-        {
-            if (callback != null)
-                callback.accept(result, failure);
-        }
-        finally
-        {
-            state(failure == null ? FINISHED : FAILED);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    Unseekables<?> keys()
-    {
-        return preLoadContext.keys();
-    }
-
-    private void fail(Throwable throwable)
-    {
-        commandStore.agent().onUncaughtException(throwable);
-        commandStore.checkInStoreThread();
-        Invariants.nonNull(throwable);
-
-        if (state.isComplete())
-            return;
-
-        try
-        {
-            switch (state)
-            {
-                case COMPLETING:
-                    break; // everything's cleaned up, invoke callback
-                case RUNNING:
-                    context.revertChanges();
-                case PREPARING:
-                    commandStore.abortCurrentOperation();
-                case LOADING:
-                    context.releaseResources(commandStore);
-                case INITIALIZED:
-                    break; // nothing to clean up, call callback
-            }
-            if (commandStore.hasSafeStore())
-                commandStore.agent().onUncaughtException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
-        }
-        catch (Throwable cleanup)
-        {
-            commandStore.agent().onUncaughtException(cleanup);
-            throwable.addSuppressed(cleanup);
-        }
-
-        finish(null, throwable);
-    }
-
-    // return true iff ready to run
-    protected boolean runInternal(boolean loadOnly)
-    {
-        switch (state)
-        {
-            default: throw new IllegalStateException("Unexpected state " + state);
-            case INITIALIZED:
-                state(LOADING);
-            case LOADING:
-                if (!loader.load(preLoadContext.primaryTxnId(), context, this::onLoaded))
-                    return false;
-                state(PREPARING);
-                if (loadOnly)
-                    return true;
-            case PREPARING:
-                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.commandsForKey, context.commandsForRanges);
-                state(RUNNING);
-            case RUNNING:
-
-                result = apply(safeStore);
-                // TODO (required): currently, we are not very efficient about ensuring that we persist the absolute minimum amount of state. Improve that.
-                List<SavedCommand.DiffWriter> diffs = null;
-                for (AccordSafeCommand commandState : context.commands.values())
-                {
-                    SavedCommand.DiffWriter diff = commandState.diff();
-                    if (diff == null)
-                        continue;
-                    if (diffs == null)
-                        diffs = new ArrayList<>(context.commands.size());
-                    diffs.add(diff);
-                    if (CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED.getBoolean())
-                    {
-                        if (sanityCheck == null)
-                            sanityCheck = new ArrayList<>(context.commands.size());
-                        sanityCheck.add(commandState.current());
-                    }
-                }
-
-                boolean flushed = false;
-                if (diffs != null || safeStore.fieldUpdates() != null)
-                {
-                    Runnable onFlush = () -> finish(result, null);
-                    if (safeStore.fieldUpdates() != null)
-                        commandStore.persistFieldUpdates(safeStore.fieldUpdates(), diffs == null ? onFlush : null);
-                    if (diffs != null)
-                        appendCommands(diffs, onFlush);
-                    flushed = true;
-                }
-
-                commandStore.completeOperation(safeStore);
-                context.releaseResources(commandStore);
-                state(COMPLETING);
-                if (flushed)
-                    return false;
-
-            case COMPLETING:
-                finish(result, null);
-            case FINISHED:
-            case FAILED:
-                break;
-        }
-
-        return false;
-    }
-
-    private void appendCommands(List<SavedCommand.DiffWriter> diffs, Runnable onFlush)
-    {
-        if (sanityCheck != null)
-        {
-            Invariants.checkState(CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED.getBoolean());
-            Condition condition = Condition.newOneTimeCondition();
-            this.commandStore.appendCommands(diffs, condition::signal);
-            condition.awaitUninterruptibly();
-
-            for (Command check : sanityCheck)
-                this.commandStore.sanityCheckCommand(check);
-
-            if (onFlush != null) onFlush.run();
-        }
-        else
-        {
-            this.commandStore.appendCommands(diffs, onFlush);
-        }
-    }
-
-    @Override
-    public void run()
-    {
-        setLoggingIds();
-        logger.trace("Running {} with state {}", this, state);
-        try
-        {
-            commandStore.checkInStoreThread();
-            commandStore.setCurrentOperation(this);
-            try
-            {
-                runInternal(false);
-            }
-            catch (Throwable t)
-            {
-                logger.error("Operation {} failed", this, t);
-                fail(t);
-            }
-            finally
-            {
-                commandStore.unsetCurrentOperation(this);
-            }
-        }
-        finally
-        {
-            logger.trace("Exiting {}", this);
-            clearLoggingIds();
-        }
-    }
-
-    private boolean preRun()
-    {
-        commandStore.checkInStoreThread();
-        try
-        {
-            return runInternal(true);
-        }
-        catch (Throwable t)
-        {
-            logger.error("Operation {} failed", this, t);
-            fail(t);
-            return false;
-        }
-    }
-
-    @Override
-    public Cancellable start(BiConsumer<? super R, Throwable> callback)
-    {
-        Invariants.checkState(this.callback == null);
-        this.callback = callback;
-        if (!commandStore.inStore() || preRun())
-            commandStore.executor().execute(this);
-        return this;
-    }
-
-    @Override
-    public void cancel()
-    {
-    }
-
-    static class ForFunction<R> extends AsyncOperation<R>
-    {
-        private final Function<? super SafeCommandStore, R> function;
-
-        public ForFunction(AccordCommandStore commandStore, PreLoadContext loadCtx, Function<? super SafeCommandStore, R> function)
-        {
-            super(commandStore, loadCtx);
-            this.function = function;
-        }
-
-        @Override
-        public R apply(SafeCommandStore commandStore)
-        {
-            return function.apply(commandStore);
-        }
-    }
-
-    public static <T> AsyncOperation<T> create(CommandStore commandStore, PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
-    {
-        return new ForFunction<>((AccordCommandStore) commandStore, loadCtx, function);
-    }
-
-    // TODO (desired): these anonymous ops are somewhat tricky to debug. We may want to at least give them names.
-    static class ForConsumer extends AsyncOperation<Void>
-    {
-        private final Consumer<? super SafeCommandStore> consumer;
-
-        public ForConsumer(AccordCommandStore commandStore, PreLoadContext loadCtx, Consumer<? super SafeCommandStore> consumer)
-        {
-            super(commandStore, loadCtx);
-            this.consumer = consumer;
-        }
-
-        @Override
-        public Void apply(SafeCommandStore commandStore)
-        {
-            consumer.accept(commandStore);
-            return null;
-        }
-    }
-
-    public static AsyncOperation<Void> create(CommandStore commandStore, PreLoadContext loadCtx, Consumer<? super SafeCommandStore> consumer)
-    {
-        return new ForConsumer((AccordCommandStore) commandStore, loadCtx, consumer);
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/accord/events/CacheEvents.java b/src/java/org/apache/cassandra/service/accord/events/CacheEvents.java
index d9e3451967..b37e418509 100644
--- a/src/java/org/apache/cassandra/service/accord/events/CacheEvents.java
+++ b/src/java/org/apache/cassandra/service/accord/events/CacheEvents.java
@@ -30,7 +30,7 @@ import jdk.jfr.StackTrace;
 @StackTrace(false)
 public abstract class CacheEvents extends Event
 {
-    public int store;
+    public int shard;
     public String instance;
     public String key;
     public String status;
diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java
index 45832e8711..c9c1b96161 100644
--- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java
+++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java
@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.service.accord.interop;
 
-import java.util.BitSet;
 import javax.annotation.Nullable;
 
 import accord.api.LocalListeners;
@@ -42,12 +41,12 @@ import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.primitives.Writes;
 import accord.topology.Topologies;
+import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType;
 import org.apache.cassandra.service.accord.serializers.ApplySerializers.ApplySerializer;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
-import org.jctools.queues.MpscChunkedArrayQueue;
 
 import static accord.utils.Invariants.checkState;
 import static com.google.common.base.Preconditions.checkArgument;
@@ -83,9 +82,9 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         }
     };
 
-    transient BitSet waitingOn;
     transient int waitingOnCount;
-    final MpscChunkedArrayQueue<LocalListeners.Registered> listeners = new MpscChunkedArrayQueue<>(4, 1 << 30);
+    transient Int2ObjectHashMap<LocalListeners.Registered> listeners;
+    boolean failed;
 
     private AccordInteropApply(Kind kind, TxnId txnId, Route<?> route, long waitForEpoch, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, @Nullable FullRoute<?> fullRoute, Writes writes, Result result)
     {
@@ -97,14 +96,6 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         super(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
     }
 
-    @Override
-    public void process()
-    {
-        waitingOn = new BitSet();
-        super.process();
-    }
-
-
     @Override
     public ApplyReply apply(SafeCommandStore safeStore, StoreParticipants participants)
     {
@@ -133,12 +124,20 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             case PreCommitted:
             case Committed:
             case PreApplied:
+                LocalListeners.Registered listener = safeStore.register(txnId, this);
                 synchronized (this)
                 {
-                    waitingOn.set(safeStore.commandStore().id());
-                    ++waitingOnCount;
+                    if (!failed)
+                    {
+                        if (listeners == null)
+                            listeners = new Int2ObjectHashMap<>();
+                        listeners.put(safeStore.commandStore().id(), listener);
+                        ++waitingOnCount;
+                        listener = null;
+                    }
                 }
-                listeners.add(safeStore.register(txnId, this));
+                if (listener != null)
+                    listener.cancel();
                 break;
 
             case Applied:
@@ -155,9 +154,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         // and prevents races where we respond before dispatching all the required reads (if the reads are
         // completing faster than the reads can be setup on all required shards)
         if (-1 == --waitingOnCount)
-        {
             node.reply(replyTo, replyContext, ApplyReply.Applied, null);
-        }
     }
 
     @Override
@@ -169,7 +166,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
     }
 
     @Override
-    public void accept(ApplyReply reply, Throwable failure)
+    protected void acceptInternal(ApplyReply reply, Throwable failure)
     {
         if (reply == ApplyReply.Insufficient)
         {
@@ -181,7 +178,7 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
         {
             node.reply(replyTo, replyContext, null, failure);
             node.agent().onUncaughtException(failure);
-            cancel();
+            fail();
         }
 
         // Unless failed always ack to indicate setup has completed otherwise the counter never gets to -1
@@ -189,11 +186,18 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             ack();
     }
 
-    private void cancel()
+    private void fail()
     {
-        listeners.drain(LocalListeners.Registered::cancel);
+        Int2ObjectHashMap<LocalListeners.Registered> listeners;
+        synchronized (this)
+        {
+            failed = true;
+            listeners = this.listeners;
+            this.listeners = null;
+        }
+        listeners.forEach((i, v) -> v.cancel());
     }
-    
+
     @Override
     public TxnId primaryTxnId()
     {
@@ -233,7 +237,6 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
     public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand)
     {
         Command command = safeCommand.current();
-
         switch (command.status())
         {
             default: throw new AssertionError();
@@ -251,7 +254,14 @@ public class AccordInteropApply extends Apply implements LocalListeners.ComplexL
             case Truncated:
         }
 
-        ack();
+        synchronized (this)
+        {
+            if (failed)
+                return false;
+
+            listeners.remove(safeStore.commandStore().id());
+            ack();
+        }
         return false;
     }
 }
diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java
index bca3b763c1..4e40531033 100644
--- a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java
+++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java
@@ -143,10 +143,11 @@ public class CommandStoreSerializers
             out.writeUnsignedVInt(t.startOwnershipEpoch);
             if (t.endOwnershipEpoch == Long.MAX_VALUE) out.writeUnsignedVInt(0L);
             else out.writeUnsignedVInt(1 + t.endOwnershipEpoch - t.startOwnershipEpoch);
+            CommandSerializers.txnId.serialize(t.locallyWitnessedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.locallyAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.locallyDecidedAndAppliedOrInvalidatedBefore, out, version);
-            CommandSerializers.txnId.serialize(t.shardAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.shardOnlyAppliedOrInvalidatedBefore, out, version);
+            CommandSerializers.txnId.serialize(t.shardAppliedOrInvalidatedBefore, out, version);
             CommandSerializers.txnId.serialize(t.gcBefore, out, version);
             CommandSerializers.txnId.serialize(t.bootstrappedAt, out, version);
             CommandSerializers.nullableTimestamp.serialize(t.staleUntilAtLeast, out, version);
@@ -160,14 +161,15 @@ public class CommandStoreSerializers
             long endEpoch = in.readUnsignedVInt();
             if (endEpoch == 0) endEpoch = Long.MAX_VALUE;
             else endEpoch = endEpoch - 1 + startEpoch;
+            TxnId locallyWitnessedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId locallyAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId locallyDecidedAndAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
-            TxnId shardAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId shardOnlyAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
+            TxnId shardAppliedOrInvalidatedBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId gcBefore = CommandSerializers.txnId.deserialize(in, version);
             TxnId bootstrappedAt = CommandSerializers.txnId.deserialize(in, version);
             Timestamp staleUntilAtLeast = CommandSerializers.nullableTimestamp.deserialize(in, version);
-            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyWitnessedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         @Override
@@ -176,10 +178,11 @@ public class CommandStoreSerializers
             long size = TokenRange.serializer.serializedSize((TokenRange) t.range, version);
             size += TypeSizes.sizeofUnsignedVInt(t.startOwnershipEpoch);
             size += TypeSizes.sizeofUnsignedVInt(t.endOwnershipEpoch == Long.MAX_VALUE ? 0 : 1 + t.endOwnershipEpoch - t.startOwnershipEpoch);
+            size += CommandSerializers.txnId.serializedSize(t.locallyWitnessedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.locallyAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.locallyDecidedAndAppliedOrInvalidatedBefore, version);
-            size += CommandSerializers.txnId.serializedSize(t.shardAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.shardOnlyAppliedOrInvalidatedBefore, version);
+            size += CommandSerializers.txnId.serializedSize(t.shardAppliedOrInvalidatedBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.gcBefore, version);
             size += CommandSerializers.txnId.serializedSize(t.bootstrappedAt, version);
             size += CommandSerializers.nullableTimestamp.serializedSize(t.staleUntilAtLeast, version);
diff --git a/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java
index c537434d95..0f109bd83d 100644
--- a/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java
+++ b/src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java
@@ -153,9 +153,9 @@ public class ReadDataSerializers
             CommandSerializers.txnId.serialize(read.txnId, out, version);
             KeySerializers.participants.serialize(read.readScope, out, version);
             out.writeUnsignedVInt(read.executeAtEpoch);
-            CommandSerializers.partialTxn.serialize(read.partialTxn, out, version);
-            DepsSerializer.partialDeps.serialize(read.partialDeps, out, version);
-            KeySerializers.fullRoute.serialize(read.route, out, version);
+            CommandSerializers.partialTxn.serialize(read.partialTxn(), out, version);
+            DepsSerializer.partialDeps.serialize(read.partialDeps(), out, version);
+            KeySerializers.fullRoute.serialize(read.route(), out, version);
         }
 
         @Override
@@ -176,9 +176,9 @@ public class ReadDataSerializers
             return CommandSerializers.txnId.serializedSize(read.txnId, version)
                    + KeySerializers.participants.serializedSize(read.readScope, version)
                    + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
-                   + CommandSerializers.partialTxn.serializedSize(read.partialTxn, version)
-                   + DepsSerializer.partialDeps.serializedSize(read.partialDeps, version)
-                   + KeySerializers.fullRoute.serializedSize(read.route, version);
+                   + CommandSerializers.partialTxn.serializedSize(read.partialTxn(), version)
+                   + DepsSerializer.partialDeps.serializedSize(read.partialDeps(), version)
+                   + KeySerializers.fullRoute.serializedSize(read.route(), version);
         }
     };
 
diff --git a/src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java b/src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java
index e88f8fbf5f..c02dc41a80 100644
--- a/src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java
+++ b/src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java
@@ -22,8 +22,6 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.SortedSet;
-import java.util.TreeSet;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -63,9 +61,10 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
 
     private Keys extractItemKeys()
     {
-        SortedSet<PartitionKey> keysSet = new TreeSet<>(Key::compareTo);
-        forEach(i -> keysSet.add(getKey(i)));
-        return new Keys(keysSet);
+        Key[] keys = new Key[size()];
+        for (int i = 0 ; i < keys.length ; ++i)
+            keys[i] = getKey(items[i]);
+        return Keys.ofSorted(keys);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java b/src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java
index b8011c68e0..e790ff6062 100644
--- a/src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java
+++ b/src/java/org/apache/cassandra/service/accord/txn/AbstractSerialized.java
@@ -71,9 +71,15 @@ public abstract class AbstractSerialized<T>
 
     protected T get()
     {
-        if (memoized == null)
-            memoized = AccordSerializers.deserialize(bytes, serializer());
-        return memoized;
+        T result = memoized;
+        if (result == null)
+            memoized = result = AccordSerializers.deserialize(bytes, serializer());
+        return result;
+    }
+
+    public void unmemoize()
+    {
+        memoized = null;
     }
 
     protected ByteBuffer bytes()
diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java
index f9d09409fb..dd8a56d119 100644
--- a/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java
+++ b/src/java/org/apache/cassandra/service/accord/txn/TxnRead.java
@@ -148,6 +148,12 @@ public class TxnRead extends AbstractKeySorted<TxnNamedRead> implements Read
         return new TxnNamedRead[size];
     }
 
+    public void unmemoize()
+    {
+        for (int i = 0 ; i < size() ; ++i)
+            items[i].unmemoize();
+    }
+
     @Override
     public Keys keys()
     {
diff --git a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java
index cc005bf6e1..f8f0dc4983 100644
--- a/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java
+++ b/src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
 import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.Write;
-import accord.impl.AbstractSafeCommandStore;
 import accord.impl.TimestampsForKey;
 import accord.impl.TimestampsForKeys;
 import accord.local.SafeCommandStore;
@@ -185,7 +184,6 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         };
     }
 
-
     /**
      * Partition update that can later be supplemented with data from the read phase
      */
@@ -373,13 +371,19 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         return new Update[size];
     }
 
+    public void unmemoize()
+    {
+        for (int i = 0 ; i < size() ; ++i)
+            items[i].unmemoize();
+    }
+
     @Override
     public AsyncChain<Void> apply(Seekable key, SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, DataStore store, PartialTxn txn)
     {
         // TODO (expected, efficiency): 99.9999% of the time we can just use executeAt.hlc(), so can avoid bringing
         //  cfk into memory by retaining at all times in memory key ranges that are dirty and must use this logic;
         //  any that aren't can just use executeAt.hlc
-        TimestampsForKey cfk = TimestampsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?,?,?>) safeStore, ((Key) key).toUnseekable(), txnId, executeAt, true);
+        TimestampsForKey cfk = TimestampsForKeys.updateLastExecutionTimestamps(safeStore, ((Key) key).toUnseekable(), txnId, executeAt, true);
         long timestamp = AccordSafeTimestampsForKey.timestampMicrosFor(cfk, executeAt, true);
         // TODO (low priority - do we need to compute nowInSeconds, or can we just use executeAt?)
         int nowInSeconds = AccordSafeTimestampsForKey.nowInSecondsFor(cfk, executeAt, true);
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index a4ba258801..478ba6ef63 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -768,7 +768,7 @@ public class ByteBufferUtil
 
     public static ByteBuffer bytes(TimeUUID uuid)
     {
-        return bytes(uuid.asUUID());
+        return ByteBuffer.wrap(UUIDGen.decompose(uuid.msb(), uuid.lsb()));
     }
 
     // Returns whether {@code prefix} is a prefix of {@code value}.
diff --git a/src/java/org/apache/cassandra/utils/UUIDGen.java b/src/java/org/apache/cassandra/utils/UUIDGen.java
index 14ab23083b..5ece1cbf83 100644
--- a/src/java/org/apache/cassandra/utils/UUIDGen.java
+++ b/src/java/org/apache/cassandra/utils/UUIDGen.java
@@ -45,13 +45,16 @@ public class UUIDGen
     /** decomposes a uuid into raw bytes. */
     public static byte[] decompose(UUID uuid)
     {
-        long most = uuid.getMostSignificantBits();
-        long least = uuid.getLeastSignificantBits();
+        return decompose(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
+    }
+
+    public static byte[] decompose(long msb, long lsb)
+    {
         byte[] b = new byte[16];
         for (int i = 0; i < 8; i++)
         {
-            b[i] = (byte)(most >>> ((7-i) * 8));
-            b[8+i] = (byte)(least >>> ((7-i) * 8));
+            b[i] = (byte)(msb >>> ((7-i) * 8));
+            b[8+i] = (byte)(lsb >>> ((7-i) * 8));
         }
         return b;
     }
diff --git a/src/java/org/apache/cassandra/utils/btree/AbstractBTreeMap.java b/src/java/org/apache/cassandra/utils/btree/AbstractBTreeMap.java
index 3abef3bca0..7ef33b417a 100644
--- a/src/java/org/apache/cassandra/utils/btree/AbstractBTreeMap.java
+++ b/src/java/org/apache/cassandra/utils/btree/AbstractBTreeMap.java
@@ -33,11 +33,13 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
 {
     protected final Object[] tree;
     protected final KeyComparator<K, V> comparator;
+    protected final AsymmetricKeyComparator<K> asymmetricComparator;
 
-    protected AbstractBTreeMap(Object[] tree, KeyComparator<K, V> comparator)
+    protected AbstractBTreeMap(Object[] tree, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricComparator)
     {
         this.tree = tree;
         this.comparator = comparator;
+        this.asymmetricComparator = asymmetricComparator;
     }
 
     /**
@@ -93,7 +95,7 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
     {
         if (key == null)
             throw new NullPointerException();
-        Entry<K, V> entry = BTree.find(tree, comparator, new Entry<>((K)key, null));
+        Entry<K, V> entry = (Entry<K, V>) BTree.find(tree, asymmetricComparator, key);
         if (entry != null)
             return entry.getValue();
         return null;
@@ -161,6 +163,22 @@ public abstract class AbstractBTreeMap<K, V> extends AbstractMap<K, V>
         }
     }
 
+    protected static class AsymmetricKeyComparator<K> implements Comparator<Object>
+    {
+        protected final Comparator<K> keyComparator;
+
+        protected AsymmetricKeyComparator(Comparator<K> keyComparator)
+        {
+            this.keyComparator = keyComparator;
+        }
+
+        @Override
+        public int compare(Object o1, Object o2)
+        {
+            return keyComparator.compare(((Map.Entry<K, ?>)o1).getKey(), (K)o2);
+        }
+    }
+
     static class Entry<K, V> extends AbstractMap.SimpleEntry<K, V>
     {
         public Entry(K key, V value)
diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeBiMap.java b/src/java/org/apache/cassandra/utils/btree/BTreeBiMap.java
index 1e1984e635..3480b8dbaa 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTreeBiMap.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTreeBiMap.java
@@ -29,16 +29,32 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 {
     private final Object[] inverse;
     private final KeyComparator<V, K> valueComparator;
+    private final AsymmetricKeyComparator<V> asymmetricValueComparator;
 
     protected static <K, V> BTreeBiMap<K, V> withComparators(Object[] tree, Object [] inverse, Comparator<K> comparator, Comparator<V> valueComparator)
     {
-        return new BTreeBiMap<>(tree, inverse, new KeyComparator<>(comparator), new KeyComparator<>(valueComparator));
+        KeyComparator<K, V> keyComparator = new KeyComparator<>(comparator);
+        AsymmetricKeyComparator<K> asymmetricKeyComparator = new AsymmetricKeyComparator<>(comparator);
+        KeyComparator<V, K> valueKeyComparator;
+        AsymmetricKeyComparator<V> asymmetricValueComparator;
+        if (comparator == valueComparator)
+        {
+            valueKeyComparator = (KeyComparator<V, K>) keyComparator;
+            asymmetricValueComparator = (AsymmetricKeyComparator<V>) asymmetricKeyComparator;
+        }
+        else
+        {
+            valueKeyComparator = new KeyComparator<>(valueComparator);
+            asymmetricValueComparator = new AsymmetricKeyComparator<>(valueComparator);
+        }
+        return new BTreeBiMap<>(tree, inverse, keyComparator, asymmetricKeyComparator, valueKeyComparator, asymmetricValueComparator);
     }
 
-    private BTreeBiMap(Object[] tree, Object [] inverse, KeyComparator<K, V> comparator, KeyComparator<V, K> valueComparator)
+    private BTreeBiMap(Object[] tree, Object [] inverse, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricKeyComparator, KeyComparator<V, K> valueComparator, AsymmetricKeyComparator<V> asymmetricValueComparator)
     {
-        super(tree, comparator);
+        super(tree, comparator, asymmetricKeyComparator);
         this.valueComparator = valueComparator;
+        this.asymmetricValueComparator = asymmetricValueComparator;
         this.inverse = inverse;
     }
 
@@ -55,7 +71,7 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
     @Override
     public BiMap<V, K> inverse()
     {
-        return new BTreeBiMap<>(inverse, tree, valueComparator, comparator);
+        return new BTreeBiMap<>(inverse, tree, valueComparator, asymmetricValueComparator, comparator, asymmetricComparator);
     }
 
     @Override
@@ -72,8 +88,8 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 
         return new BTreeBiMap<>(BTree.update(tree, new Object[]{ entry }, comparator, UpdateFunction.noOp()),
                                 BTree.update(inverse, new Object[] { new AbstractBTreeMap.Entry<>(value, key) }, valueComparator, UpdateFunction.noOp()),
-                                comparator,
-                                valueComparator);
+                                comparator, asymmetricComparator,
+                                valueComparator, asymmetricValueComparator);
     }
 
     @Override
@@ -92,7 +108,7 @@ public class BTreeBiMap<K, V> extends AbstractBTreeMap<K, V> implements BiMap<K,
 
         Object[] newTree = BTreeRemoval.remove(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
         Object[] newInverse = BTreeRemoval.remove(inverse, valueComparator, new AbstractBTreeMap.Entry<>(existingEntry.getValue(), null));
-        return new BTreeBiMap<>(newTree, newInverse, comparator, valueComparator);
+        return new BTreeBiMap<>(newTree, newInverse, comparator, asymmetricComparator, valueComparator, asymmetricValueComparator);
     }
 
     public Set<V> values()
diff --git a/src/java/org/apache/cassandra/utils/btree/BTreeMap.java b/src/java/org/apache/cassandra/utils/btree/BTreeMap.java
index 2d8e92a298..c27cf2125b 100644
--- a/src/java/org/apache/cassandra/utils/btree/BTreeMap.java
+++ b/src/java/org/apache/cassandra/utils/btree/BTreeMap.java
@@ -33,12 +33,12 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
 {
     protected static <K, V> BTreeMap<K, V> withComparator(Object[] tree, Comparator<K> comparator)
     {
-        return new BTreeMap<>(tree, new KeyComparator<>(comparator));
+        return new BTreeMap<>(tree, new KeyComparator<>(comparator), new AsymmetricKeyComparator<>(comparator));
     }
 
-    protected BTreeMap(Object[] tree, KeyComparator<K, V> comparator)
+    protected BTreeMap(Object[] tree, KeyComparator<K, V> comparator, AsymmetricKeyComparator<K> asymmetricComparator)
     {
-        super(tree, comparator);
+        super(tree, comparator, asymmetricComparator);
     }
 
     public static <K, V> BTreeMap<K, V> empty(Comparator<K> comparator)
@@ -61,7 +61,7 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         AbstractBTreeMap.Entry<K, V> existing;
         if ((existing = BTree.find(tree, comparator, entry)) != null && !existing.equals(entry))
             throw new IllegalStateException("Map already contains " + key);
-        return new BTreeMap<>(BTree.update(tree, new Object[]{ entry }, comparator, UpdateFunction.noOp()), comparator);
+        return new BTreeMap<>(BTree.update(tree, BTree.singleton(entry), comparator, UpdateFunction.noOp()), comparator, asymmetricComparator);
     }
 
     public BTreeMap<K, V> withForce(K key, V value)
@@ -69,7 +69,7 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         if (key == null || value == null)
             throw new NullPointerException();
         AbstractBTreeMap.Entry<K, V> entry = new AbstractBTreeMap.Entry<>(key, value);
-        return new BTreeMap<>(BTree.update(tree, new Object[] { entry }, comparator, UpdateFunction.Simple.of((a, b) -> b)), comparator);
+        return new BTreeMap<>(BTree.update(tree, BTree.singleton(entry), comparator, UpdateFunction.Simple.of((a, b) -> b)), comparator, asymmetricComparator);
     }
 
     public BTreeMap<K, V> without(K key)
@@ -77,13 +77,14 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
         if (key == null)
             throw new NullPointerException();
 
-        return new BTreeMap<>(BTreeRemoval.remove(tree, comparator, new AbstractBTreeMap.Entry<>(key, null)), comparator);
+        return new BTreeMap<>(BTreeRemoval.remove(tree, asymmetricComparator, key), comparator, asymmetricComparator);
     }
 
     @Override
     public Map.Entry<K, V> lowerEntry(K key)
     {
-        return BTree.lower(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.lower(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -96,7 +97,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> floorEntry(K key)
     {
-        return BTree.floor(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.floor(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -109,7 +111,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> ceilingEntry(K key)
     {
-        return BTree.ceil(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.ceil(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -122,7 +125,8 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public Map.Entry<K, V> higherEntry(K key)
     {
-        return BTree.higher(tree, comparator, new AbstractBTreeMap.Entry<>(key, null));
+        //noinspection unchecked
+        return (Map.Entry<K, V>) BTree.higher(tree, asymmetricComparator, key);
     }
 
     @Override
@@ -151,8 +155,9 @@ public class BTreeMap<K, V> extends AbstractBTreeMap<K, V> implements NavigableM
     @Override
     public NavigableMap<K, V> descendingMap()
     {
+        Comparator<K> reversed = comparator.keyComparator.reversed();
         return new BTreeMap<>(BTree.build(BulkIterator.of(BTree.iterable(tree, BTree.Dir.DESC).iterator()), BTree.size(tree), UpdateFunction.noOp),
-                              new KeyComparator<>(comparator.keyComparator.reversed()));
+                              new KeyComparator<>(reversed), new AsymmetricKeyComparator<>(reversed));
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java b/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java
new file mode 100644
index 0000000000..5632a10770
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/ConcurrentLinkedStack.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public class ConcurrentLinkedStack<T>
+{
+    static final class Node<T> extends IntrusiveStack<Node<T>>
+    {
+        final T value;
+        Node(T value)
+        {
+            this.value = value;
+        }
+    }
+
+    private volatile Node<T> head;
+    private static final AtomicReferenceFieldUpdater<ConcurrentLinkedStack, Node> headUpdater = AtomicReferenceFieldUpdater.newUpdater(ConcurrentLinkedStack.class, Node.class, "head");
+
+    public void push(T value)
+    {
+        IntrusiveStack.push(headUpdater, this, (Node)new Node<>(value));
+    }
+
+    public boolean isEmpty()
+    {
+        return head == null;
+    }
+
+    public void drain(Consumer<T> forEach, boolean reverse)
+    {
+        if (isEmpty())
+            return;
+
+        Node<T> head = headUpdater.getAndSet(this, null);
+        if (reverse) head = IntrusiveStack.reverse(head);
+        IntrusiveStack.forEach(head, n -> n.value, forEach);
+    }
+
+    public <P> void drain(BiConsumer<P, T> forEach, P param, boolean reverse)
+    {
+        if (isEmpty())
+            return;
+
+        Node<T> head = headUpdater.getAndSet(this, null);
+        if (reverse) head = IntrusiveStack.reverse(head);
+        IntrusiveStack.forEach(head, n -> n.value, forEach, param);
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java
index e61d56545f..5e59de3b17 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/IntrusiveStack.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent;
 
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -189,10 +190,25 @@ public class IntrusiveStack<T extends IntrusiveStack<T>> implements Iterable<T>
     }
 
     protected static <T extends IntrusiveStack<T>> void forEach(T list, Consumer<? super T> forEach)
+    {
+        forEach(list, Function.identity(), forEach);
+    }
+
+    protected static <T extends IntrusiveStack<T>, P> void forEach(T list, BiConsumer<P, ? super T> forEach, P param)
+    {
+        forEach(list, Function.identity(), forEach, param);
+    }
+
+    protected static <T extends IntrusiveStack<T>, V> void forEach(T list, Function<? super T, ? extends V> getter, Consumer<? super V> forEach)
+    {
+        forEach(list, getter, Consumer::accept, forEach);
+    }
+
+    protected static <P, T extends IntrusiveStack<T>, V> void forEach(T list, Function<? super T, ? extends V> getter, BiConsumer<? super P, ? super V> forEach, P param)
     {
         while (list != null)
         {
-            forEach.accept(list);
+            forEach.accept(param, getter.apply(list));
             list = list.next;
         }
     }
diff --git a/src/java/org/apache/cassandra/utils/concurrent/LockWithAsyncSignal.java b/src/java/org/apache/cassandra/utils/concurrent/LockWithAsyncSignal.java
new file mode 100644
index 0000000000..e64a87221f
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/concurrent/LockWithAsyncSignal.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils.concurrent;
+
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.LockSupport;
+
+import accord.utils.Invariants;
+
+// WARNING: experimental - needs more testing
+public class LockWithAsyncSignal implements Lock
+{
+    interface AwaitFunction<T extends Throwable>
+    {
+        T await(LockWithAsyncSignal lock, Waiter waiter) throws T;
+    }
+
+    private static final Waiter AWAITING_LOCK = new Waiter(0, null);
+
+    private volatile Thread owner;
+    private static final AtomicReferenceFieldUpdater<LockWithAsyncSignal, Thread> ownerUpdater = AtomicReferenceFieldUpdater.newUpdater(LockWithAsyncSignal.class, Thread.class, "owner");
+    private int depth;
+
+    // TODO (desired): better combined queue
+    final ConcurrentSkipListSet<Waiter> waiters = new ConcurrentSkipListSet<>();
+
+    static class Waiter implements Comparable<Waiter>
+    {
+        final long ticket;
+        final Thread thread;
+
+        Waiter(long ticket, Thread thread)
+        {
+            this.ticket = ticket;
+            this.thread = thread;
+        }
+
+        @Override
+        public int compareTo(Waiter that)
+        {
+            return Long.compare(this.ticket, that.ticket);
+        }
+    }
+
+    volatile int signal;
+    private static final AtomicIntegerFieldUpdater<LockWithAsyncSignal> signalUpdater = AtomicIntegerFieldUpdater.newUpdater(LockWithAsyncSignal.class, "signal");
+
+    volatile long ticket;
+    private static final AtomicLongFieldUpdater<LockWithAsyncSignal> ticketUpdater = AtomicLongFieldUpdater.newUpdater(LockWithAsyncSignal.class, "ticket");
+
+    public void lock()
+    {
+        lockInternal(LockWithAsyncSignal::awaitUninterruptibly);
+    }
+
+    public void lockInterruptibly() throws InterruptedException
+    {
+        lockInternal(LockWithAsyncSignal::awaitThrows);
+    }
+
+    private <T extends Throwable> void lockInternal(AwaitFunction<T> await) throws T
+    {
+        Thread thread = Thread.currentThread();
+        if (ownerUpdater.compareAndSet(this, null, thread) || owner == thread)
+        {
+            ++depth;
+        }
+        else
+        {
+            awaitLock(false, thread, 1, await);
+        }
+    }
+
+    public boolean tryLock()
+    {
+        Thread thread = Thread.currentThread();
+        if (!ownerUpdater.compareAndSet(this, null, thread) && owner != thread)
+            return false;
+
+        ++depth;
+        return true;
+    }
+
+    public void await() throws InterruptedException
+    {
+        Thread thread = Thread.currentThread();
+        int restoreDepth = depth;
+        Invariants.checkState(owner == thread);
+
+        depth = 0;
+        owner = null;
+
+        awaitLock(true, thread, restoreDepth, LockWithAsyncSignal::awaitDeferThrow);
+    }
+
+    public void unlock()
+    {
+        Invariants.checkState(owner == Thread.currentThread());
+        if (--depth > 0)
+            return;
+
+        owner = null;
+        wakeOne();
+    }
+
+    private <T extends Throwable> void awaitLock(boolean awaitingSignal, Thread thread, int restoreDepth, AwaitFunction<T> await) throws T
+    {
+        T pending = null;
+        while (true)
+        {
+            Waiter waiter = register(awaitingSignal, thread);
+            if (awaitingSignal && signal == 0)
+            {
+                if (owner == null)
+                    wakeOne(false); // will not wake ourselves as we only signal pure lock waiters
+            }
+            else if (ownerUpdater.compareAndSet(this, null, thread))
+            {
+                depth = restoreDepth;
+                waiters.remove(waiter);
+                if (pending != null)
+                    throw pending;
+                return;
+            }
+            pending = firstNonNull(pending, await.await(this, waiter));
+            awaitingSignal &= pending == null;
+        }
+    }
+
+    private static <T> T firstNonNull(T cur, T next)
+    {
+        return cur != null ? cur : next;
+    }
+
+    public void signal()
+    {
+        if (signalUpdater.compareAndSet(this, 0, 1) && owner == null)
+            wakeOne(true);
+    }
+
+    public void clearSignal()
+    {
+        signal = 0;
+    }
+
+    public boolean isOwner(Thread thread)
+    {
+        return thread == owner;
+    }
+
+    private Waiter register(boolean awaitingSignal, Thread thread)
+    {
+        long ticket = ticketUpdater.updateAndGet(this, v -> v == Long.MAX_VALUE ? 1 : v + 1);
+        if (awaitingSignal)
+            ticket = -ticket;
+        Waiter waiter = new Waiter(ticket, thread);
+        waiters.add(waiter);
+        return waiter;
+    }
+
+    private InterruptedException awaitDeferThrow(Waiter waiter)
+    {
+        while (waiters.contains(waiter))
+        {
+            if (Thread.interrupted())
+            {
+                waiters.remove(waiter);
+                return new InterruptedException();
+            }
+            LockSupport.park();
+        }
+        return null;
+    }
+
+    private InterruptedException awaitThrows(Waiter waiter) throws InterruptedException
+    {
+        while (waiters.contains(waiter))
+        {
+            if (Thread.interrupted())
+            {
+                if (!waiters.remove(waiter))
+                    wakeOne(waiter.ticket < 0 || signal > 0);
+
+                throw new InterruptedException();
+            }
+            LockSupport.park();
+        }
+        return null;
+    }
+
+    private RuntimeException awaitUninterruptibly(Waiter waiter)
+    {
+        while (waiters.contains(waiter))
+            LockSupport.park();
+        return null;
+    }
+
+    private void wakeOne()
+    {
+        wakeOne(signal > 0);
+    }
+
+    private void wakeOne(boolean awaitingSignal)
+    {
+        Waiter wake;
+        if (awaitingSignal)
+        {
+            wake = waiters.pollFirst();
+            if (wake == null)
+                return;
+        }
+        else
+        {
+            do
+            {
+                wake = waiters.ceiling(AWAITING_LOCK);
+                if (wake == null)
+                    return;
+            } while (!waiters.remove(wake));
+        }
+
+        LockSupport.unpark(wake.thread);
+    }
+
+    @Override
+    public boolean tryLock(long time, TimeUnit unit) throws InterruptedException
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Condition newCondition()
+    {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/src/java/org/apache/cassandra/utils/concurrent/Ref.java b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
index 6f0836b3e3..6719948e33 100644
--- a/src/java/org/apache/cassandra/utils/concurrent/Ref.java
+++ b/src/java/org/apache/cassandra/utils/concurrent/Ref.java
@@ -156,6 +156,11 @@ public final class Ref<T> implements RefCounted<T>
         return referent;
     }
 
+    public Tidy tidier()
+    {
+        return state.globalState.tidy;
+    }
+
     public Ref<T> tryRef()
     {
         return state.globalState.ref() ? new Ref<>(referent, state.globalState) : null;
diff --git a/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java
new file mode 100644
index 0000000000..88a785d9a9
--- /dev/null
+++ b/test/burn/org/apache/cassandra/service/accord/AccordExecutorBurnTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+import org.junit.Test;
+
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.utils.concurrent.Semaphore;
+import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+public class AccordExecutorBurnTest
+{
+    static class State
+    {
+        final AccordExecutor executor;
+        final AccordCommandStore[] commandStores;
+        final ExecutorPlus loadGen;
+        final int generators;
+        final int targetCount;
+        final Semaphore permits;
+        final AtomicInteger submitted = new AtomicInteger();
+        final AtomicInteger completed = new AtomicInteger();
+
+        State(AccordExecutor executor, Function<AccordExecutor, AccordCommandStore> storeFactory,
+              int taskCount, int concurrency, int generators, int commandStores)
+        {
+            this.executor = executor;
+            this.targetCount = taskCount;
+            this.permits = Semaphore.newSemaphore(concurrency);
+            this.generators = generators;
+            this.loadGen = executorFactory().pooled("loadgen", generators);
+            this.commandStores = new AccordCommandStore[commandStores];
+            for (int i = 0 ; i < commandStores ; ++i)
+                this.commandStores[i] = storeFactory.apply(executor);
+        }
+
+        void start()
+        {
+            for (int i = 0 ; i < generators ; ++i)
+                loadGen.execute(this::run);
+        }
+
+        void run()
+        {
+            while (true)
+            {
+                int slot = submitted.get();
+                if (slot >= targetCount)
+                    return;
+                if (!submitted.compareAndSet(slot, slot + 1))
+                    continue;
+
+                try { permits.acquire(1); }
+                catch (InterruptedException e) { throw new UncheckedInterruptedException(e); }
+                submitSomething();
+            }
+        }
+
+        private void submitSomething()
+        {
+        }
+    }
+
+    @Test
+    public void test()
+    {
+
+    }
+
+}
diff --git a/test/conf/cassandra-jmx-disabled-sslconfig.yaml b/test/conf/cassandra-jmx-disabled-sslconfig.yaml
index 12baefb75f..4708909c96 100644
--- a/test/conf/cassandra-jmx-disabled-sslconfig.yaml
+++ b/test/conf/cassandra-jmx-disabled-sslconfig.yaml
@@ -147,3 +147,9 @@ stream_throughput_outbound: 24MiB/s
 sasi_indexes_enabled: true
 materialized_views_enabled: true
 file_cache_enabled: true
+
+accord:
+  enabled: true
+  journal_directory: build/test/cassandra/accord_journal
+  queue_shard_count: 2
+  command_store_shard_count: 4
\ No newline at end of file
diff --git a/test/conf/cassandra-jmx-pem-sslconfig.yaml b/test/conf/cassandra-jmx-pem-sslconfig.yaml
index 55adfd87dd..0e8f204791 100644
--- a/test/conf/cassandra-jmx-pem-sslconfig.yaml
+++ b/test/conf/cassandra-jmx-pem-sslconfig.yaml
@@ -147,3 +147,9 @@ stream_throughput_outbound: 24MiB/s
 sasi_indexes_enabled: true
 materialized_views_enabled: true
 file_cache_enabled: true
+
+accord:
+  enabled: true
+  journal_directory: build/test/cassandra/accord_journal
+  queue_shard_count: 2
+  command_store_shard_count: 4
\ No newline at end of file
diff --git a/test/conf/cassandra-jmx-sslconfig-with-passwordfile.yaml b/test/conf/cassandra-jmx-sslconfig-with-passwordfile.yaml
index 0b495485d1..6f66b04bbc 100644
--- a/test/conf/cassandra-jmx-sslconfig-with-passwordfile.yaml
+++ b/test/conf/cassandra-jmx-sslconfig-with-passwordfile.yaml
@@ -72,3 +72,9 @@ stream_throughput_outbound: 24MiB/s
 sasi_indexes_enabled: true
 materialized_views_enabled: true
 file_cache_enabled: true
+
+accord:
+    enabled: true
+    journal_directory: build/test/cassandra/accord_journal
+    queue_shard_count: 2
+    command_store_shard_count: 4
\ No newline at end of file
diff --git a/test/conf/cassandra-jmx-sslconfig.yaml b/test/conf/cassandra-jmx-sslconfig.yaml
index 1a6ef9a945..cd2f9fb036 100644
--- a/test/conf/cassandra-jmx-sslconfig.yaml
+++ b/test/conf/cassandra-jmx-sslconfig.yaml
@@ -72,3 +72,9 @@ stream_throughput_outbound: 24MiB/s
 sasi_indexes_enabled: true
 materialized_views_enabled: true
 file_cache_enabled: true
+
+accord:
+    enabled: true
+    journal_directory: build/test/cassandra/accord_journal
+    queue_shard_count: 2
+    command_store_shard_count: 4
\ No newline at end of file
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index 84d9478d40..20c1d2fd70 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -68,6 +68,12 @@ local_read_size_fail_threshold: 8192KiB
 row_index_read_size_warn_threshold: 4096KiB
 row_index_read_size_fail_threshold: 8192KiB
 
+accord:
+    enabled: true
+    journal_directory: build/test/cassandra/accord_journal
+    queue_shard_count: 2
+    command_store_shard_count: 4
+
 memtable:
     configurations:
         skiplist:
@@ -114,8 +120,3 @@ memtable:
             class_name: TrieMemtable
 # Note: keep the memtable configuration at the end of the file, so that the default mapping can be changed without
 # duplicating the whole section above.
-
-accord:
-    enabled: true
-    journal_directory: build/test/cassandra/accord_journal
-    shard_count: 4
diff --git a/test/distributed/org/apache/cassandra/distributed/Cluster.java b/test/distributed/org/apache/cassandra/distributed/Cluster.java
index 4effc4e9c4..07e64041c6 100644
--- a/test/distributed/org/apache/cassandra/distributed/Cluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/Cluster.java
@@ -76,6 +76,11 @@ public class Cluster extends AbstractCluster<IInvokableInstance>
         }
     }
 
+    public void forEach(IIsolatedExecutor.SerializableRunnable runnable)
+    {
+        forEach(i -> i.runOnInstance(runnable));
+    }
+
     public void enableMessageLogging()
     {
         filters().allVerbs().inbound().messagesMatching((from, to, msg) -> {
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index 13a7dae535..3ca044df00 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.distributed.impl;
 
 import java.io.IOException;
 import java.lang.annotation.Annotation;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -61,9 +62,11 @@ import org.junit.Assume;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.concurrent.ExecutorFactory;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.Constants;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.Feature;
@@ -92,6 +95,7 @@ import org.apache.cassandra.io.util.PathUtils;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Isolated;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Shared;
 import org.apache.cassandra.utils.Shared.Recursive;
 import org.apache.cassandra.utils.concurrent.Condition;
@@ -167,7 +171,6 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
     private final INodeProvisionStrategy.Factory nodeProvisionStrategy;
     private final IInstanceInitializer instanceInitializer;
     private final int datadirCount;
-    private volatile Thread.UncaughtExceptionHandler previousHandler = null;
     private volatile BiPredicate<Integer, Throwable> ignoreUncaughtThrowable = null;
     private final List<Throwable> uncaughtExceptions = new CopyOnWriteArrayList<>();
 
@@ -193,6 +196,29 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
             CassandraRelevantProperties.TEST_FLUSH_LOCAL_SCHEMA_CHANGES.reset();
             CassandraRelevantProperties.NON_GRACEFUL_SHUTDOWN.reset();
             CassandraRelevantProperties.IO_NETTY_TRANSPORT_NONATIVE.setBoolean(false);
+            withInstanceInitializer((classLoader, threadGroup, i, i1) -> {
+                try
+                {
+                    Class<?> ef = classLoader.loadClass(ExecutorFactory.class.getName());
+                    Class<?> efd = classLoader.loadClass(ExecutorFactory.Default.class.getName());
+                    Constructor<?> newEfd = efd.getConstructor(ClassLoader.class, ThreadGroup.class, Thread.UncaughtExceptionHandler.class);
+                    Object executorFactory = newEfd.newInstance(classLoader, threadGroup, threadGroup);
+                    Class<?> efg = classLoader.loadClass(ExecutorFactory.Global.class.getName());
+                    Method setEfg = efg.getMethod("unsafeSet", ef);
+                    setEfg.invoke(null, executorFactory);
+                }
+                catch (ClassNotFoundException e)
+                {
+                    if (this instanceof Cluster.Builder)
+                        throw new RuntimeException(e);
+                    else
+                        logger.info("Unable to set ExecutorFactory for instance {}", i, e);
+                }
+                catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            });
         }
 
         public AbstractBuilder(Factory<I, C, B> factory)
@@ -306,7 +332,31 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
             ++generation;
             IClassTransformer transformer = classTransformer == null ? null : classTransformer.initialise();
             ClassLoader classLoader = new InstanceClassLoader(generation, config.num(), version.classpath, sharedClassLoader, sharedClassPredicate, transformer);
-            ThreadGroup threadGroup = new ThreadGroup(clusterThreadGroup, "node" + config.num() + (generation > 1 ? "_" + generation : ""));
+            Consumer<Throwable> stabilityInspector;
+            {
+                try
+                {
+                    Class<?> owner = classLoader.loadClass(JVMStabilityInspector.class.getName());
+                    Method method = owner.getMethod("inspectThrowable", Throwable.class);
+                    stabilityInspector = t -> {
+                        try { method.invoke(null, t); }
+                        catch (IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); }
+                    };
+                }
+                catch (ClassNotFoundException | NoSuchMethodException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }
+            ThreadGroup threadGroup = new ThreadGroup(clusterThreadGroup, "node" + config.num() + (generation > 1 ? "_" + generation : ""))
+            {
+                @Override
+                public void uncaughtException(Thread t, Throwable e)
+                {
+                    AbstractCluster.this.uncaughtException(t, e);
+                    stabilityInspector.accept(e);
+                }
+            };
             if (instanceInitializer != null)
                 instanceInitializer.initialise(classLoader, threadGroup, config.num(), generation);
 
@@ -771,11 +821,6 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
         }
     }
 
-    public void forEach(IIsolatedExecutor.SerializableRunnable runnable)
-    {
-        forEach(i -> i.sync(runnable));
-    }
-
     public void forEach(Consumer<? super I> consumer)
     {
         forEach(instances, consumer);
@@ -1034,8 +1079,8 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
 
     public void startup()
     {
-        previousHandler = Thread.getDefaultUncaughtExceptionHandler();
-        Thread.setDefaultUncaughtExceptionHandler(this::uncaughtExceptions);
+        // start the JNA cleaner on the system class loader to avoid pinning an instance
+        com.sun.jna.internal.Cleaner.getCleaner();
         try (AllMembersAliveMonitor monitor = new AllMembersAliveMonitor())
         {
             monitor.startPolling();
@@ -1072,19 +1117,12 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
         }
     }
 
-    private void uncaughtExceptions(Thread thread, Throwable error)
+    private void uncaughtException(Thread thread, Throwable error)
     {
         if (!(thread.getContextClassLoader() instanceof InstanceClassLoader))
-        {
-            Thread.UncaughtExceptionHandler handler = previousHandler;
-            if (null != handler)
-                handler.uncaughtException(thread, error);
             return;
-        }
 
         InstanceClassLoader cl = (InstanceClassLoader) thread.getContextClassLoader();
-        get(cl.getInstanceId()).uncaughtException(thread, error);
-
         BiPredicate<Integer, Throwable> ignore = ignoreUncaughtThrowable;
         I instance = get(cl.getInstanceId());
         if ((ignore == null || !ignore.test(cl.getInstanceId(), error)) && instance != null && !instance.isShutdown())
@@ -1133,8 +1171,6 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
             PathUtils.deleteRecursive(root);
         else
             logger.error("Not removing directories, as some instances haven't fully stopped.");
-        Thread.setDefaultUncaughtExceptionHandler(previousHandler);
-        previousHandler = null;
         checkAndResetUncaughtExceptions();
         //checkForThreadLeaks();
         //withThreadLeakCheck(futures);
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
index 281b95c0b0..132ece309b 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java
@@ -57,7 +57,6 @@ import org.apache.cassandra.audit.AuditLogManager;
 import org.apache.cassandra.auth.AuthCache;
 import org.apache.cassandra.batchlog.Batch;
 import org.apache.cassandra.batchlog.BatchlogManager;
-import org.apache.cassandra.concurrent.ExecutorFactory;
 import org.apache.cassandra.concurrent.ExecutorLocals;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
@@ -200,6 +199,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
     private volatile boolean internodeMessagingStarted = false;
     private final AtomicLong startedAt = new AtomicLong();
     private IsolatedJmx isolatedJmx;
+    private static boolean RECEIVE_MESSAGES_ASYNC = false;
+    public static void setReceiveMessagesAsync(boolean v) {RECEIVE_MESSAGES_ASYNC = v; }
 
     /** @deprecated See CASSANDRA-17013 */
     @Deprecated(since = "4.1")
@@ -215,7 +216,12 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
 
     Instance(IInstanceConfig config, ClassLoader classLoader, FileSystem fileSystem, ShutdownExecutor shutdownExecutor)
     {
-        super("node" + config.num(), classLoader, executorFactory().pooled("isolatedExecutor", Integer.MAX_VALUE), shutdownExecutor);
+        super("node" + config.num(), classLoader, executorFactory().configurePooled("isolatedExecutor", Integer.MAX_VALUE)
+                                                                   // we report uncaught exceptions on node thread pools, but
+                                                                   // we never reported exceptions from this thread pool, and
+                                                                   // tests deliberately produce a lot so it would be a lot of
+                                                                   // work to whitelist the exceptions - volunteers welcome!
+                                                                   .withUncaughtExceptionHandler(JVMStabilityInspector::uncaughtException).build(), shutdownExecutor);
         this.config = config;
         if (fileSystem != null)
             File.unsafeSetFilesystem(fileSystem);
@@ -399,6 +405,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
         });
     }
 
+    // TODO (desired): remove this method; no longer needed
     public void uncaughtException(Thread thread, Throwable throwable)
     {
         sync(JVMStabilityInspector::uncaughtException).accept(thread, throwable);
@@ -509,8 +516,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
     @Override
     public void receiveMessage(IMessage message)
     {
-        sync(receiveMessageRunnable(message)).accept(false);
-//        async(receiveMessageRunnable(message)).apply(false);
+        if (RECEIVE_MESSAGES_ASYNC) async(receiveMessageRunnable(message)).apply(false);
+        else sync(receiveMessageRunnable(message)).accept(false);
     }
 
     @Override
@@ -562,7 +569,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                 }
                 // This can cause deadlocks when sending messages to self so use Stage.MISC.executor() just to have a
                 // place for it to run
-                if ( executor == ImmediateExecutor.INSTANCE)
+                if (executor == ImmediateExecutor.INSTANCE)
                     executor = Stage.MISC.executor();
                 executor.execute(ExecutorLocals.create(state), () -> MessagingService.instance().inboundSink.accept(messageIn));
             }
@@ -645,9 +652,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
                 {
                     assert config.networkTopology().contains(config.broadcastAddress()) : String.format("Network topology %s doesn't contain the address %s",
                                                                                                         config.networkTopology(), config.broadcastAddress());
-                    // org.apache.cassandra.distributed.impl.AbstractCluster.startup sets the exception handler for the thread
-                    // so extract it to populate ExecutorFactory.Global
-                    ExecutorFactory.Global.tryUnsafeSet(new ExecutorFactory.Default(Thread.currentThread().getContextClassLoader(), null, Thread.getDefaultUncaughtExceptionHandler()));
                     DistributedTestInitialLocationProvider.assign(config.networkTopology());
                     CassandraDaemon.getInstanceForTesting().activate(false);
                     // TODO: filters won't work for the messages dispatched during startup
@@ -741,9 +745,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
 
     protected void partialStartup(ICluster<?> cluster) throws IOException, NoSuchFieldException, IllegalAccessException, ExecutionException, InterruptedException, StartupException
     {
-        // org.apache.cassandra.distributed.impl.AbstractCluster.startup sets the exception handler for the thread
-        // so extract it to populate ExecutorFactory.Global
-        ExecutorFactory.Global.tryUnsafeSet(new ExecutorFactory.Default(Thread.currentThread().getContextClassLoader(), null, Thread.getDefaultUncaughtExceptionHandler()));
         if (config.has(GOSSIP))
         {
             // TODO: hacky
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
index 81deb3e2fe..37a1eb0c61 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
@@ -99,7 +99,8 @@ public class InstanceConfig implements IInstanceConfig
                 .set("cdc_raw_directory", cdc_raw_directory)
                 .set("accord.enabled", accord.enabled)
                 .set("accord.journal_directory", accord.journal_directory)
-                .set("accord.shard_count", accord.shard_count.toString())
+                .set("accord.queue_shard_count", accord.queue_shard_count.toString())
+                .set("accord.command_store_shard_count", accord.command_store_shard_count.toString())
                 .set("accord.recover_delay", accord.recover_delay.toString())
                 .set("partitioner", "org.apache.cassandra.dht.Murmur3Partitioner")
                 .set("start_native_transport", true)
@@ -327,7 +328,8 @@ public class InstanceConfig implements IInstanceConfig
         AccordSpec accordSpec = new AccordSpec();
         accordSpec.enabled = DTEST_ACCORD_ENABLED.getBoolean();
         accordSpec.journal_directory = String.format("%s/node%d/accord_journal", root, nodeNum);
-        accordSpec.shard_count = new OptionaldPositiveInt(4);
+        accordSpec.queue_shard_count = new OptionaldPositiveInt(2);
+        accordSpec.command_store_shard_count = new OptionaldPositiveInt(4);
         return new InstanceConfig(nodeNum,
                                   networkTopology,
                                   provisionStrategy.ipAddress(nodeNum),
diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
index 14ac84b404..a4549788f8 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/TestBaseImpl.java
@@ -66,7 +66,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.api.NodeToolResult;
 import org.apache.cassandra.distributed.shared.DistributedTestBase;
-import org.apache.cassandra.service.accord.AccordStateCache;
+import org.apache.cassandra.service.accord.AccordCache;
 
 import static java.lang.System.currentTimeMillis;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -93,7 +93,7 @@ public class TestBaseImpl extends DistributedTestBase
         CassandraRelevantProperties.SIMULATOR_STARTED.setString(Long.toString(MILLISECONDS.toSeconds(currentTimeMillis())));
         ICluster.setup();
         SKIP_GC_INSPECTOR.setBoolean(true);
-        AccordStateCache.validateLoadOnEvict(true);
+        AccordCache.validateLoadOnEvict(true);
     }
 
     @Override
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java
index fa93afa2ae..519fee327d 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java
@@ -85,6 +85,8 @@ public class AccordBootstrapTest extends TestBaseImpl
     {
         IInstanceConfig config = cluster.newInstanceConfig();
         config.set("auto_bootstrap", true);
+        config.set("accord.shard_durability_target_splits", "1");
+        config.set("accord.shard_durability_cycle", "20s");
         IInvokableInstance newInstance = cluster.bootstrap(config);
         newInstance.startup(cluster);
         // todo: re-add once we fix write survey/join ring = false mode
@@ -121,7 +123,7 @@ public class AccordBootstrapTest extends TestBaseImpl
         try
         {
             AccordConfigurationService configService = service().configurationService();
-            boolean completed = configService.localSyncNotified(epoch).await(5, TimeUnit.SECONDS);
+            boolean completed = configService.localSyncNotified(epoch).await(30, TimeUnit.SECONDS);
             Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s",
                                             epoch, FBUtilities.getBroadcastAddressAndPort()), completed);
         }
@@ -174,7 +176,11 @@ public class AccordBootstrapTest extends TestBaseImpl
                                       .withoutVNodes()
                                       .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount))
                                       .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0"))
-                                      .withConfig(config -> config.set("accord.shard_count", 2).with(NETWORK, GOSSIP))
+                                      .withConfig(config -> config.set("accord.command_store_shard_count", 2)
+                                                                  .set("accord.queue_shard_count", 2)
+                                                                  .set("accord.shard_durability_cycle", "20s")
+                                                                  .set("accord.shard_durability_target_splits", "1")
+                                                                  .with(NETWORK, GOSSIP))
                                       .start())
         {
             long initialMax = maxEpoch(cluster);
@@ -351,7 +357,10 @@ public class AccordBootstrapTest extends TestBaseImpl
                                       .withoutVNodes()
                                       .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3))
                                       .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0"))
-                                      .withConfig(config -> config.with(NETWORK, GOSSIP))
+                                      .withConfig(config -> config
+                                                            .set("accord.shard_durability_target_splits", "1")
+                                                            .set("accord.shard_durability_cycle", "20s")
+                                                            .with(NETWORK, GOSSIP))
                                       .start())
         {
             long initialMax = maxEpoch(cluster);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java
index 09a445b30a..6a0f918572 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java
@@ -128,7 +128,7 @@ public class AccordDropTableBase extends TestBaseImpl
             inst.runOnInstance(() -> {
                 TableId tableId = TableId.fromUUID(UUID.fromString(s));
                 AccordService accord = (AccordService) AccordService.instance();
-                PreLoadContext ctx = PreLoadContext.contextFor(Ranges.single(TokenRange.fullRange(tableId)), KeyHistory.COMMANDS);
+                PreLoadContext ctx = PreLoadContext.contextFor(Ranges.single(TokenRange.fullRange(tableId)), KeyHistory.SYNC);
                 CommandStores stores = accord.node().commandStores();
                 for (int storeId : stores.ids())
                 {
@@ -137,7 +137,7 @@ public class AccordDropTableBase extends TestBaseImpl
                         AccordSafeCommandStore safe = (AccordSafeCommandStore) input;
                         for (RoutingKey key : safe.commandsForKeysKeys())
                         {
-                            AccordSafeCommandsForKey safeCFK = safe.maybeCommandsForKey(key);
+                            AccordSafeCommandsForKey safeCFK = (AccordSafeCommandsForKey) safe.ifLoadedAndInitialised(key);
                             if (safeCFK == null) // we read and found a key, but its null at load time... so ignore it
                                 continue;
                             CommandsForKey cfk = safeCFK.current();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordHostReplacementTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordHostReplacementTest.java
index 0105ad7eae..c5e4366dae 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordHostReplacementTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordHostReplacementTest.java
@@ -53,11 +53,12 @@ public class AccordHostReplacementTest extends TestBaseImpl
     {
         // start 3 node cluster, then do a host replacement of one of the nodes
         Cluster.Builder clusterBuilder = Cluster.build(3)
-                .withConfig(c -> c.with(Feature.values())
-                        .set("accord.command_store_shard_count", "1")
-                        .set("write_request_timeout", "10s")
-                        .set("read_request_timeout", "10s")
-                        .set("accord.queue_shard_count", "1")
+                                                .withConfig(c -> c.with(Feature.values())
+                                                                  .set("accord.command_store_shard_count", "1")
+                                                                  .set("write_request_timeout", "10s")
+                                                                  .set("read_request_timeout", "10s")
+                                                                  .set("accord.command_store_shard_count", "1")
+                                                                  .set("accord.queue_shard_count", "1")
                 );
         TokenSupplier tokenRing = TokenSupplier.evenlyDistributedTokens(3, clusterBuilder.getTokenCount());
         int nodeToReplace = 2;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
index c563650d72..40e60b222e 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
@@ -70,7 +70,7 @@ import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
-import static accord.local.KeyHistory.COMMANDS;
+import static accord.local.KeyHistory.SYNC;
 import static java.lang.String.format;
 
 public class AccordIncrementalRepairTest extends AccordTestBase
@@ -227,9 +227,9 @@ public class AccordIncrementalRepairTest extends AccordTestBase
     {
         Node node = accordService().node();
         AtomicReference<TxnId> waitFor = new AtomicReference<>(null);
-        AsyncChains.awaitUninterruptibly(node.commandStores().ifLocal(PreLoadContext.contextFor(key, COMMANDS), key.toUnseekable(), 0, Long.MAX_VALUE, safeStore -> {
+        AsyncChains.awaitUninterruptibly(node.commandStores().ifLocal(PreLoadContext.contextFor(key, SYNC), key.toUnseekable(), 0, Long.MAX_VALUE, safeStore -> {
             AccordSafeCommandStore store = (AccordSafeCommandStore) safeStore;
-            SafeCommandsForKey safeCfk = store.maybeCommandsForKey(key);
+            SafeCommandsForKey safeCfk = store.ifLoadedAndInitialised(key);
             if (safeCfk == null)
                 return;
             CommandsForKey cfk = safeCfk.current();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java
index 572ffaae49..144fcdf311 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.distributed.test.accord;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
 
 import org.junit.BeforeClass;
@@ -27,11 +28,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.impl.progresslog.DefaultProgressLogs;
-import accord.messages.Commit;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.api.IMessageFilters;
-import org.apache.cassandra.distributed.impl.Instance;
-import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.Verb;
 
 public class AccordIntegrationTest extends AccordTestBase
@@ -71,7 +69,6 @@ public class AccordIntegrationTest extends AccordTestBase
             lostApply.off();
             lostCommit.off();
 
-            // Querying again should trigger recovery...
             query = "BEGIN TRANSACTION\n" +
                     "  LET row1 = (SELECT v FROM " + qualifiedAccordTableName + " WHERE k=0 AND c=0);\n" +
                     "  SELECT row1.v;\n" +
@@ -105,12 +102,8 @@ public class AccordIntegrationTest extends AccordTestBase
         pauseSimpleProgressLog();
         test(cluster -> {
             // It's expected that the required Read will happen regardless of whether this fails to return a read
-            cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).messagesMatching((from, to, iMessage) -> cluster.get(from).callOnInstance(() -> {
-                Message<?> msg = Instance.deserializeMessage(iMessage);
-                if (msg.payload instanceof Commit)
-                    return ((Commit) msg.payload).readData != null;
-                return false;
-            })).drop();
+            final AtomicBoolean droppedOne = new AtomicBoolean();
+            cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).messagesMatching((from, to, iMessage) -> !droppedOne.getAndSet(true)).drop();
 
             String query = "BEGIN TRANSACTION\n" +
                            "  LET row1 = (SELECT * FROM " + qualifiedAccordTableName + " WHERE k = 0 AND c = 0);\n" +
@@ -119,6 +112,7 @@ public class AccordIntegrationTest extends AccordTestBase
                            "    INSERT INTO " + qualifiedAccordTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
                            "  END IF\n" +
                            "COMMIT TRANSACTION";
+
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] { null }, query);
 
             String check = "BEGIN TRANSACTION\n" +
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java
index 19c774a5f4..f77bfa2016 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordJournalIntegrationTest.java
@@ -111,6 +111,7 @@ public class AccordJournalIntegrationTest extends TestBaseImpl
             cluster.get(1).startup();
 
             Object[][] after = cluster.coordinator(1).execute("SELECT * FROM " + TABLE + " WHERE k = ?;", ConsistencyLevel.SERIAL, 1);
+            Assert.assertEquals(before.length, after.length);
             for (int i = 0; i < before.length; i++)
             {
                 Assert.assertTrue(Arrays.equals(before[i], after[i]));
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java
index 8f2dabe624..6ad2f09d2a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordLoadTest.java
@@ -42,6 +42,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.ICoordinator;
@@ -70,7 +71,7 @@ public class AccordLoadTest extends AccordTestBase
                                                                             .with(Feature.NETWORK, Feature.GOSSIP)
                                                                             .set("accord.shard_durability_target_splits", "64")
                                                                             .set("accord.shard_durability_cycle", "5m")
-                                                                            .set("accord.ephemeral_read_enabled", "true")
+//                                                                            .set("accord.ephemeral_read_enabled", "true")
                                                                             .set("accord.gc_delay", "5s")), 3);
     }
 
@@ -78,192 +79,190 @@ public class AccordLoadTest extends AccordTestBase
     @Test
     public void testLoad() throws Exception
     {
-        test("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'",
-             cluster -> {
+        Cluster cluster = SHARED_CLUSTER;
+        cluster.schemaChange("CREATE TABLE " + qualifiedAccordTableName + " (k int, v int, PRIMARY KEY(k)) WITH transactional_mode = 'full'");
 
-                try
+        try
+        {
+
+            final ConcurrentHashMap<Verb, AtomicInteger> verbs = new ConcurrentHashMap<>();
+            cluster.filters().outbound().messagesMatching(new IMessageFilters.Matcher()
+            {
+                @Override
+                public boolean matches(int i, int i1, IMessage iMessage)
                 {
+                    verbs.computeIfAbsent(Verb.fromId(iMessage.verb()), ignore -> new AtomicInteger()).incrementAndGet();
+                    return false;
+                }
+            }).drop();
 
-                    final ConcurrentHashMap<Verb, AtomicInteger> verbs = new ConcurrentHashMap<>();
-                    cluster.filters().outbound().messagesMatching(new IMessageFilters.Matcher()
+            ICoordinator coordinator = cluster.coordinator(1);
+            final int repairInterval = Integer.MAX_VALUE;
+            //                 final int repairInterval = 3000;
+            final int compactionInterval = Integer.MAX_VALUE;
+//                     final int compactionInterval = 3000;
+            final int flushInterval = Integer.MAX_VALUE;
+//                     final int flushInterval = 1000;
+            final int compactionPeriodSeconds = 1;
+            final int restartInterval = 150_000_000;
+            final int batchSizeLimit = 1000;
+            final long batchTime = TimeUnit.SECONDS.toNanos(10);
+            final int concurrency = 100;
+            final int ratePerSecond = 1000;
+            final int keyCount = 10_000;
+            final float readChance = 0.33f;
+            long nextRepairAt = repairInterval;
+            long nextCompactionAt = compactionInterval;
+            long nextFlushAt = flushInterval;
+            long nextRestartAt = restartInterval;
+            final ExecutorService restartExecutor = Executors.newSingleThreadExecutor();
+            final BitSet initialised = new BitSet();
+
+            cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success();
+            cluster.forEach(i -> i.runOnInstance(() -> {
+                if (compactionPeriodSeconds > 0)
+                    ((AccordService) AccordService.instance()).journal().compactor().updateCompactionPeriod(1, SECONDS);
+                //                     ((AccordSpec.JournalSpec)((AccordService) AccordService.instance()).journal().configuration()).segmentSize = 128 << 10;
+            }));
+
+            Random random = new Random();
+            //                 CopyOnWriteArrayList<Throwable> exceptions = new CopyOnWriteArrayList<>();
+            final Semaphore inFlight = new Semaphore(concurrency);
+            final RateLimiter rateLimiter = RateLimiter.create(ratePerSecond);
+            //                 long testStart = System.nanoTime();
+            //                 while (NANOSECONDS.toMinutes(System.nanoTime() - testStart) < 10 && exceptions.size() < 10000)
+            while (true)
+            {
+                final EstimatedHistogram histogram = new EstimatedHistogram(200);
+                long batchStart = System.nanoTime();
+                long batchEnd = batchStart + batchTime;
+                int batchSize = 0;
+                while (batchSize < batchSizeLimit)
+                {
+                    inFlight.acquire();
+                    rateLimiter.acquire();
+                    long commandStart = System.nanoTime();
+                    int k = random.nextInt(keyCount);
+                    if (random.nextFloat() < readChance)
                     {
-                        @Override
-                        public boolean matches(int i, int i1, IMessage iMessage)
-                        {
-                            verbs.computeIfAbsent(Verb.fromId(iMessage.verb()), ignore -> new AtomicInteger()).incrementAndGet();
-                            return false;
-                        }
-                    }).drop();
-
-                     ICoordinator coordinator = cluster.coordinator(1);
-                     final int repairInterval = Integer.MAX_VALUE;
-    //                 final int repairInterval = 3000;
-//                     final int compactionInterval = Integer.MAX_VALUE;
-                     final int compactionInterval = 3000;
-//                     final int flushInterval = Integer.MAX_VALUE;
-                     final int flushInterval = 1000;
-                     final int compactionPeriodSeconds = 1;
-                     final int restartInterval = 150_000_000;
-                     final int batchSizeLimit = 1000;
-                     final long batchTime = TimeUnit.SECONDS.toNanos(10);
-                     final int concurrency = 100;
-                     final int ratePerSecond = 1000;
-                     final int keyCount = 1_000_000;
-                     final float readChance = 0.33f;
-                     long nextRepairAt = repairInterval;
-                     long nextCompactionAt = compactionInterval;
-                     long nextFlushAt = flushInterval;
-                     long nextRestartAt = restartInterval;
-                     final ExecutorService restartExecutor = Executors.newSingleThreadExecutor();
-                     final BitSet initialised = new BitSet();
-
-                     cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success();
-                     cluster.forEach(i -> i.runOnInstance(() -> {
-                         if (compactionPeriodSeconds > 0)
-                            ((AccordService) AccordService.instance()).journal().compactor().updateCompactionPeriod(1, SECONDS);
-    //                     ((AccordSpec.JournalSpec)((AccordService) AccordService.instance()).journal().configuration()).segmentSize = 128 << 10;
-                     }));
-
-                     Random random = new Random();
-    //                 CopyOnWriteArrayList<Throwable> exceptions = new CopyOnWriteArrayList<>();
-                     final Semaphore inFlight = new Semaphore(concurrency);
-                     final RateLimiter rateLimiter = RateLimiter.create(ratePerSecond);
-    //                 long testStart = System.nanoTime();
-    //                 while (NANOSECONDS.toMinutes(System.nanoTime() - testStart) < 10 && exceptions.size() < 10000)
-                     while (true)
-                     {
-                         final EstimatedHistogram histogram = new EstimatedHistogram(200);
-                         long batchStart = System.nanoTime();
-                         long batchEnd = batchStart + batchTime;
-                         int batchSize = 0;
-                         while (batchSize < batchSizeLimit)
-                         {
-                             inFlight.acquire();
-                             rateLimiter.acquire();
-                             long commandStart = System.nanoTime();
-                             int k = random.nextInt(keyCount);
-                             if (random.nextFloat() < readChance)
-                             {
-                                 coordinator.executeWithResult((success, fail) -> {
-                                     inFlight.release();
-                                     if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
-                                     //                             else exceptions.add(fail);
-                                 }, "SELECT * FROM " + qualifiedAccordTableName + " WHERE k = ?;", ConsistencyLevel.SERIAL, k);
-                             }
-                             else if (initialised.get(k))
-                             {
-                                 coordinator.executeWithResult((success, fail) -> {
-                                     inFlight.release();
-                                     if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
-        //                             else exceptions.add(fail);
-                                 }, "UPDATE " + qualifiedAccordTableName + " SET v += 1 WHERE k = ? IF EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, k);
-                             }
-                             else
-                             {
-                                 initialised.set(k);
-                                 coordinator.executeWithResult((success, fail) -> {
-                                     inFlight.release();
-                                     if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
-                                     //                             else exceptions.add(fail);
-                                 }, "UPDATE " + qualifiedAccordTableName + " SET v = 0 WHERE k = ? IF NOT EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, k);
-                             }
-                             batchSize++;
-                             if (System.nanoTime() >= batchEnd)
-                                 break;
-                         }
-
-                         if ((nextRepairAt -= batchSize) <= 0)
-                         {
-                             nextRepairAt += repairInterval;
-                             System.out.println("repairing...");
-                             cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName);
-                         }
-
-                         if ((nextCompactionAt -= batchSize) <= 0)
-                         {
-                             nextCompactionAt += compactionInterval;
-                             System.out.println("compacting accord...");
-                             cluster.forEach(i -> {
-                                 i.nodetool("compact", "system_accord.journal");
-                                 i.runOnInstance(() -> {
-                                     ((AccordService) AccordService.instance()).journal().checkAllCommands();
-                                 });
-                             });
-                         }
-
-                         if ((nextFlushAt -= batchSize) <= 0)
-                         {
-                             nextFlushAt += flushInterval;
-                             System.out.println("flushing journal...");
-                             cluster.forEach(i -> i.runOnInstance(() -> {
-                                 ((AccordService) AccordService.instance()).journal().closeCurrentSegmentForTestingIfNonEmpty();
-                                 ((AccordService) AccordService.instance()).journal().checkAllCommands();
-                             }));
-                         }
-
-                         if ((nextRestartAt -= batchSize) <= 0)
-                         {
-                             nextRestartAt += restartInterval;
-                             int nodeIdx = random.nextInt(cluster.size());
-
-                             restartExecutor.submit(() -> {
-                                 System.out.printf("restarting node %d...\n", nodeIdx);
-                                 try
-                                 {
-                                     cluster.get(nodeIdx).shutdown().get();
-                                     cluster.get(nodeIdx).startup();
-                                     return null;
-                                 }
-                                 catch (InterruptedException | ExecutionException e)
-                                 {
-                                     throw new RuntimeException(e);
-                                 }
-                             });
-                         }
-
-                         final Date date = new Date();
-                         System.out.printf("%tT rate: %.2f/s (%d total)\n", date, (((float)batchSizeLimit * 1000) / NANOSECONDS.toMillis(System.nanoTime() - batchStart)), batchSize);
-                         System.out.printf("%tT percentiles: %d %d %d %d\n", date, histogram.percentile(.25)/1000, histogram.percentile(.5)/1000, histogram.percentile(.75)/1000, histogram.percentile(1)/1000);
-
-                         class VerbCount
-                         {
-                             final Verb verb;
-                             final int count;
-
-                             VerbCount(Verb verb, int count)
-                             {
-                                 this.verb = verb;
-                                 this.count = count;
-                             }
-                         }
-                         List<VerbCount> verbCounts = new ArrayList<>();
-                         for (Map.Entry<Verb, AtomicInteger> e : verbs.entrySet())
-                         {
-                             int count = e.getValue().getAndSet(0);
-                             if (count != 0) verbCounts.add(new VerbCount(e.getKey(), count));
-                         }
-                         verbCounts.sort(Comparator.comparing(v -> -v.count));
-
-                         StringBuilder verbSummary = new StringBuilder();
-                         for (VerbCount vs : verbCounts)
-                         {
-                             {
-                                 if (verbSummary.length() > 0)
-                                     verbSummary.append(", ");
-                                 verbSummary.append(vs.verb);
-                                 verbSummary.append(": ");
-                                 verbSummary.append(vs.count);
-                             }
-                         }
-                         System.out.printf("%tT verbs: %s\n", date, verbSummary);
-                     }
+                        coordinator.executeWithResult((success, fail) -> {
+                            inFlight.release();
+                            if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
+                            //                             else exceptions.add(fail);
+                        }, "SELECT * FROM " + qualifiedAccordTableName + " WHERE k = ?;", ConsistencyLevel.SERIAL, k);
+                    }
+                    else if (initialised.get(k))
+                    {
+                        coordinator.executeWithResult((success, fail) -> {
+                            inFlight.release();
+                            if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
+                            //                             else exceptions.add(fail);
+                        }, "UPDATE " + qualifiedAccordTableName + " SET v += 1 WHERE k = ? IF EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, k);
+                    }
+                    else
+                    {
+                        initialised.set(k);
+                        coordinator.executeWithResult((success, fail) -> {
+                            inFlight.release();
+                            if (fail == null) histogram.add(NANOSECONDS.toMicros(System.nanoTime() - commandStart));
+                            //                             else exceptions.add(fail);
+                        }, "UPDATE " + qualifiedAccordTableName + " SET v = 0 WHERE k = ? IF NOT EXISTS;", ConsistencyLevel.SERIAL, ConsistencyLevel.QUORUM, k);
+                    }
+                    batchSize++;
+                    if (System.nanoTime() >= batchEnd)
+                        break;
                 }
-                catch (Throwable t)
+
+                if ((nextRepairAt -= batchSize) <= 0)
                 {
-                    t.printStackTrace();
+                    nextRepairAt += repairInterval;
+                    System.out.println("repairing...");
+                    cluster.coordinator(1).instance().nodetool("repair", qualifiedAccordTableName);
                 }
-             }
-        );
+
+                if ((nextCompactionAt -= batchSize) <= 0)
+                {
+                    nextCompactionAt += compactionInterval;
+                    System.out.println("compacting accord...");
+                    cluster.forEach(i -> {
+                        i.nodetool("compact", "system_accord.journal");
+                        i.runOnInstance(() -> {
+                            ((AccordService) AccordService.instance()).journal().checkAllCommands();
+                        });
+                    });
+                }
+
+                if ((nextFlushAt -= batchSize) <= 0)
+                {
+                    nextFlushAt += flushInterval;
+                    System.out.println("flushing journal...");
+                    cluster.forEach(i -> i.runOnInstance(() -> {
+                        ((AccordService) AccordService.instance()).journal().closeCurrentSegmentForTestingIfNonEmpty();
+                        ((AccordService) AccordService.instance()).journal().checkAllCommands();
+                    }));
+                }
+
+                if ((nextRestartAt -= batchSize) <= 0)
+                {
+                    nextRestartAt += restartInterval;
+                    int nodeIdx = random.nextInt(cluster.size());
+
+                    restartExecutor.submit(() -> {
+                        System.out.printf("restarting node %d...\n", nodeIdx);
+                        try
+                        {
+                            cluster.get(nodeIdx).shutdown().get();
+                            cluster.get(nodeIdx).startup();
+                            return null;
+                        }
+                        catch (InterruptedException | ExecutionException e)
+                        {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+                final Date date = new Date();
+                System.out.printf("%tT rate: %.2f/s (%d total)\n", date, (((float)batchSizeLimit * 1000) / NANOSECONDS.toMillis(System.nanoTime() - batchStart)), batchSize);
+                System.out.printf("%tT percentiles: %d %d %d %d\n", date, histogram.percentile(.25)/1000, histogram.percentile(.5)/1000, histogram.percentile(.75)/1000, histogram.percentile(1)/1000);
+
+                class VerbCount
+                {
+                    final Verb verb;
+                    final int count;
+
+                    VerbCount(Verb verb, int count)
+                    {
+                        this.verb = verb;
+                        this.count = count;
+                    }
+                }
+                List<VerbCount> verbCounts = new ArrayList<>();
+                for (Map.Entry<Verb, AtomicInteger> e : verbs.entrySet())
+                {
+                    int count = e.getValue().getAndSet(0);
+                    if (count != 0) verbCounts.add(new VerbCount(e.getKey(), count));
+                }
+                verbCounts.sort(Comparator.comparing(v -> -v.count));
+
+                StringBuilder verbSummary = new StringBuilder();
+                for (VerbCount vs : verbCounts)
+                {
+                    {
+                        if (verbSummary.length() > 0)
+                            verbSummary.append(", ");
+                        verbSummary.append(vs.verb);
+                        verbSummary.append(": ");
+                        verbSummary.append(vs.count);
+                    }
+                }
+                System.out.printf("%tT verbs: %s\n", date, verbSummary);
+            }
+        }
+        catch (Throwable t)
+        {
+            t.printStackTrace();
+        }
     }
 
     @Override
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java
index 7a0c2cdfb9..f08176046b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java
@@ -21,6 +21,9 @@ package org.apache.cassandra.distributed.test.accord;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
 import org.junit.Before;
@@ -29,6 +32,7 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.IMessageFilters;
 import org.apache.cassandra.distributed.api.Row;
@@ -120,10 +124,16 @@ public class AccordMetricsTest extends AccordTestBase
     @Test
     public void testPreemptionMetrics()
     {
-        IMessageFilters.Filter commitFilter1 = SHARED_CLUSTER.filters().outbound().verbs(Verb.ACCORD_COMMIT_REQ.id).from(1).to(1).drop();
-        IMessageFilters.Filter commitFilter2 = SHARED_CLUSTER.filters().outbound().verbs(Verb.ACCORD_COMMIT_REQ.id).from(1).to(2).drop();
-        commitFilter1.on();
-        commitFilter2.on();
+        ScheduledExecutorService exec = Executors.newScheduledThreadPool(1);
+        IMessageFilters.Matcher delay = (from, to, m) -> {
+            exec.schedule(() -> SHARED_CLUSTER.get(to).receiveMessageWithInvokingThread(m), 10L, TimeUnit.SECONDS);
+            return true;
+        };
+        IMessageFilters.Filter preacceptDelay = SHARED_CLUSTER.filters().outbound().verbs(Verb.ACCORD_PRE_ACCEPT_REQ.id).from(1).to(1)
+                                                            .messagesMatching(delay)
+                                                            .drop();
+
+        SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverDelay(100L, TimeUnit.MILLISECONDS));
 
         countingMetrics0 = getMetrics();
         try
@@ -136,10 +146,10 @@ public class AccordMetricsTest extends AccordTestBase
             Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(WritePreemptedException.class));
         }
 
-        assertCoordinatorMetrics(0, "rw", 1, 0, 1, 0, 0);
-        assertCoordinatorMetrics(1, "rw", 0, 0, 0, 0, 1);
-        assertReplicaMetrics(0, "rw", 1, 1, 1);
-        assertReplicaMetrics(1, "rw", 1, 1, 1);
+        assertCoordinatorMetrics(0, "rw", 0, 0, 1, 0, 0);
+        assertCoordinatorMetrics(1, "rw", 0, 0, 0, 0, 0);
+        assertReplicaMetrics(0, "rw", 0, 0, 0);
+        assertReplicaMetrics(1, "rw", 0, 0, 0);
 
         assertZeroMetrics("ro");
 
@@ -154,12 +164,15 @@ public class AccordMetricsTest extends AccordTestBase
             Assertions.assertThat(ex).is(AssertionUtils.rootCauseIs(ReadPreemptedException.class));
         }
 
-        assertCoordinatorMetrics(0, "ro", 1, 0, 1, 0, 0);
-        assertCoordinatorMetrics(1, "ro", 0, 0, 0, 0, 1);
-        assertReplicaMetrics(0, "ro", 1, 1, 0);
-        assertReplicaMetrics(1, "ro", 1, 1, 0);
+        assertCoordinatorMetrics(0, "ro", 0, 0, 1, 0, 0);
+        assertCoordinatorMetrics(1, "ro", 0, 0, 0, 0, 0);
+        assertReplicaMetrics(0, "ro", 0, 0, 0);
+        assertReplicaMetrics(1, "ro", 0, 0, 0);
 
         assertZeroMetrics("rw");
+        SHARED_CLUSTER.forEach(() -> DatabaseDescriptor.setAccordRecoverDelay(10L, TimeUnit.SECONDS));
+        preacceptDelay.off();
+        exec.shutdown();
     }
 
     @Test
@@ -269,7 +282,7 @@ public class AccordMetricsTest extends AccordTestBase
 
         // Verify that per-store global cache stats are published to the appropriate virtual table:
         SimpleQueryResult storeCacheResults = SHARED_CLUSTER.get(node + 1)
-                                                   .executeInternalWithResult("SELECT * FROM system_views.accord_command_store_cache");
+                                                   .executeInternalWithResult("SELECT * FROM system_views.accord_executor_cache");
         assertThat(storeCacheResults).hasNext();
     }
 
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationRaceTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationRaceTestBase.java
index 0503684e11..98e12e416a 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationRaceTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationRaceTestBase.java
@@ -217,6 +217,9 @@ public abstract class AccordMigrationRaceTestBase extends AccordTestBase
         // Otherwise repair complains if you don't specify a keyspace
         CassandraRelevantProperties.SYSTEM_TRACES_DEFAULT_RF.setInt(3);
         AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("paxos_variant", PaxosVariant.v2.name())
+                                                                                    .set("accord.shard_durability_cycle", "1m")
+                                                                                    .set("accord.shard_durability_target_splits", "1")
+                                                                                    .set("accord.shard_durability_cycle", "60s")
                                                                                     .set("write_request_timeout", "2s")
                                                                                     .set("accord.range_migration", "explicit")), 3);
         partitioner = FBUtilities.newPartitioner(SHARED_CLUSTER.get(1).callsOnInstance(() -> DatabaseDescriptor.getPartitioner().getClass().getSimpleName()).call());
@@ -642,11 +645,12 @@ public abstract class AccordMigrationRaceTestBase extends AccordTestBase
                      outOfSyncInstance.runOnInstance(() -> HintsService.instance.resumeDispatch());
                      // The initial hinting attempt should fail, unless it's a batchlog routing failure in which
                      // case the coordinator has already caught up so the hint will succeed on the first try
-                     // Can only really have this case for BATCHLOG_FAILED_TIMEOUT_THEN_HINT becuase Accord timeouts don't
+                     // Can only really have this case for BATCHLOG_FAILED_TIMEOUT_THEN_HINT because Accord timeouts don't
                      // write hints so there is nothing to test
                      if (migrateAwayFromAccord && scenario == BATCHLOG_FAILED_TIMEOUT_THEN_HINT)
                      {
                          Callable<Boolean> test = () -> outOfSyncInstance.callOnInstance(() -> {
+                             HintsService.instance.flushAndFsyncBlockingly();
                              logger.info("startingAccordTimeouts {}, startingAccordPreempts {}, startingAccordMigrationRejects {}, startingHintTimeouts {}, accord timeouts {}, accordPreempts {}, accordMigrationRejects {}, hint timeouts {}", startingAccordTimeouts, startingAccordPreempted, startingAccordMigrationRejects, startingHintTimeouts, ClientRequestsMetricsHolder.accordWriteMetrics.timeouts.getCount(), ClientRequestsMetricsHolder.accordWriteMetrics.preempted.getCount(), ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.getCount(), HintsServiceMetrics.hintsTimedOut.getCount());
                              AccordClientRequestMetrics accordMetrics = ClientRequestsMetricsHolder.accordWriteMetrics;
                              return accordMetrics.timeouts.getCount() >= (startingAccordTimeouts + 1) && HintsServiceMetrics.hintsTimedOut.getCount() >= (startingHintTimeouts + 1);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java
index 73c15d99ce..284436f178 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java
@@ -345,35 +345,33 @@ public class AccordMigrationTest extends AccordTestBase
                 List<byte[]> keys = expectedMigrations.stream().map(p -> p.left.array()).collect(Collectors.toList());
                 List<Integer> intKeys = expectedMigrations.stream().map(p -> ByteBufferUtil.toInt(p.left)).collect(Collectors.toList());
                 List<UUID> tables = expectedMigrations.stream().map(p -> p.right).collect(Collectors.toList());
-                for (int i = 1; i < SHARED_CLUSTER.size(); i++)
-                {
-                    int instanceIndex = i;
-                    IInvokableInstance instance = SHARED_CLUSTER.get(i);
-                    instance.runOnInstance(() -> {
-                        Map<Pair<ByteBuffer, UUID>, ConsensusMigratedAt> cacheMap = ConsensusKeyMigrationState.MIGRATION_STATE_CACHE.asMap();
-                        String cacheMessage = format("Instance %d Expected %s migrations but found in cache %s", instanceIndex, intKeys, cacheMap);
-                        assertEquals(cacheMessage, keys.size(), cacheMap.size());
-                        for (int j = 0; j < keys.size(); j++)
-                        {
-                            assertTrue(cacheMessage,
-                                       cacheMap.containsKey(Pair.create(ByteBuffer.wrap(keys.get(j)), tables.get(j))));
-                        }
+                // Notification of all replicas that the key was migrated was removed so they will each have to run
+                // a local barrier first to find out the key was migrated. Not sure if we will add it back somehow.
+                IInvokableInstance instance = SHARED_CLUSTER.get(1);
+                instance.runOnInstance(() -> {
+                    Map<Pair<ByteBuffer, UUID>, ConsensusMigratedAt> cacheMap = ConsensusKeyMigrationState.MIGRATION_STATE_CACHE.asMap();
+                    String cacheMessage = format("Instance %d Expected %s migrations but found in cache %s", 1, intKeys, cacheMap);
+                    assertEquals(cacheMessage, keys.size(), cacheMap.size());
+                    for (int j = 0; j < keys.size(); j++)
+                    {
+                        assertTrue(cacheMessage,
+                                   cacheMap.containsKey(Pair.create(ByteBuffer.wrap(keys.get(j)), tables.get(j))));
+                    }
 
-                        UntypedResultSet result = QueryProcessor.executeInternal("SELECT * from " + SYSTEM_KEYSPACE_NAME + "." + CONSENSUS_MIGRATION_STATE);
-                        String tableMessage = format("Instance %d Expected %s migrations but found in system table %s", instanceIndex, intKeys, result);
-                        assertEquals(tableMessage, keys.size(), result.size());
-                        Iterator<UntypedResultSet.Row> resultIterator = result.iterator();
-                        for (int j = 0; j < result.size(); j++)
-                        {
-                            UntypedResultSet.Row row = resultIterator.next();
-                            boolean foundKey = false;
-                            for (byte[] expectedKey : keys)
-                                if (ByteBuffer.wrap(expectedKey).equals(row.getBytes("row_key")))
-                                    foundKey = true;
-                            assertTrue(tableMessage, foundKey);
-                        }
-                    });
-                }
+                    UntypedResultSet result = QueryProcessor.executeInternal("SELECT * from " + SYSTEM_KEYSPACE_NAME + "." + CONSENSUS_MIGRATION_STATE);
+                    String tableMessage = format("Instance %d Expected %s migrations but found in system table %s", 1, intKeys, result);
+                    assertEquals(tableMessage, keys.size(), result.size());
+                    Iterator<UntypedResultSet.Row> resultIterator = result.iterator();
+                    for (int j = 0; j < result.size(); j++)
+                    {
+                        UntypedResultSet.Row row = resultIterator.next();
+                        boolean foundKey = false;
+                        for (byte[] expectedKey : keys)
+                            if (ByteBuffer.wrap(expectedKey).equals(row.getBytes("row_key")))
+                                foundKey = true;
+                        assertTrue(tableMessage, foundKey);
+                    }
+                });
             }
             catch (Throwable t)
             {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java
index e326f49d25..545892f696 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordProgressLogTest.java
@@ -42,20 +42,20 @@ public class AccordProgressLogTest extends TestBaseImpl
     @Test
     public void testRecoveryTimeWindow() throws Throwable
     {
-        try (Cluster cluster = init(Cluster.build(3)
+        try (Cluster cluster = init(Cluster.build(2)
                                            .withoutVNodes()
                                            .withConfig(c -> c.with(Feature.NETWORK)
                                                              .set("accord.enabled", "true")
-                                                            .set("accord.recover_delay", "1s"))
+                                                             .set("accord.recover_delay", "1s"))
                                            .start()))
         {
             cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}");
             cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c)) WITH " + TransactionalMode.full.asCqlParam());
             String query = "BEGIN TRANSACTION\n" +
-                           "  SELECT * FROM ks.tbl WHERE k=0 AND c=0;\n" +
+                           "  INSERT INTO ks.tbl (k, c) VALUES (0, 0);\n" +
                            "COMMIT TRANSACTION";
 
-            IMessageFilters.Filter dropCommit = cluster.filters().outbound().from(1).verbs(Verb.ACCORD_COMMIT_REQ.id).drop();
+            IMessageFilters.Filter dropPreAccept = cluster.filters().outbound().from(1).to(2).verbs(Verb.ACCORD_PRE_ACCEPT_REQ.id).drop();
             AtomicLong recoveryStartedAt = new AtomicLong();
             Semaphore waitForRecovery = new Semaphore(0);
             IMessageFilters.Filter recovery = cluster.filters().outbound().messagesMatching((from, to, message) -> {
@@ -69,27 +69,21 @@ public class AccordProgressLogTest extends TestBaseImpl
 
             long coordinationStartedAt = System.nanoTime();
             boolean failed = false;
-            try
-            {
-                cluster.coordinator(1).executeWithResult(query, ConsistencyLevel.ANY);
-            }
-            catch (Throwable e)
-            {
-                failed = true;
-            }
+            try { cluster.coordinator(1).executeWithResult(query, ConsistencyLevel.ANY); }
+            catch (Throwable e) { failed = true; }
             Assert.assertTrue(failed);
 
             waitForRecovery.acquire();
             long timeDeltaMillis = TimeUnit.NANOSECONDS.toMillis(recoveryStartedAt.get() - coordinationStartedAt);
             Assert.assertTrue("Recovery started in " + timeDeltaMillis + "ms", timeDeltaMillis >= 1000);
-            Assert.assertTrue("Recovery started in " + timeDeltaMillis + "ms", timeDeltaMillis <= 3000);
+            Assert.assertTrue("Recovery started in " + timeDeltaMillis + "ms", timeDeltaMillis <= 5000);
         }
     }
 
     @Test
     public void testFetchTimeWindow() throws Throwable
     {
-        try (Cluster cluster = init(Cluster.build(3)
+        try (Cluster cluster = init(Cluster.build(2)
                                            .withoutVNodes()
                                            .withConfig(c -> c.with(Feature.NETWORK).set("accord.enabled", "true"))
                                            .start()))
@@ -97,7 +91,7 @@ public class AccordProgressLogTest extends TestBaseImpl
             cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}");
             cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c)) WITH " + TransactionalMode.full.asCqlParam());
             String query = "BEGIN TRANSACTION\n" +
-                           "  SELECT * FROM ks.tbl WHERE k=0 AND c=0;\n" +
+                           "  INSERT INTO ks.tbl (k, c) VALUES (0, 0);\n" +
                            "COMMIT TRANSACTION";
 
             IMessageFilters.Filter dropApply = cluster.filters().outbound().from(1).verbs(Verb.ACCORD_APPLY_REQ.id).drop();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java
index b3e2407af2..79109cf202 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java
@@ -339,7 +339,11 @@ public abstract class AccordTestBase extends TestBaseImpl
                                                            .set("write_request_timeout", "10s")
                                                            .set("transaction_timeout", "15s")
                                                            .set("native_transport_timeout", "30s")
-                                                           .set("accord.shard_count", "2"))
+                                                           .set("accord.ephemeral_read_enabled", "false")
+                                                           .set("accord.shard_durability_target_splits", "1")
+                                                           .set("accord.shard_durability_cycle", "60s")
+                                                           .set("accord.command_store_shard_count", "2")
+                                                           .set("accord.queue_shard_count", "2"))
                                          .withInstanceInitializer(EnforceUpdateDoesNotPerformRead::install);
         builder = options.apply(builder);
         return init(builder.start());
diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java
index ae269dbdd6..bcc02b89a6 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java
@@ -85,9 +85,10 @@ public class BootWithMetadataTest extends TestBaseImpl
                 assertEquals(1, ClusterMetadata.current().fullCMSMembers().size());
                 assertTrue(ClusterMetadata.current().fullCMSMembers().contains(InetAddressAndPort.getByNameUnchecked("127.0.0.1")));
                 Keyspace ks = Keyspace.open(KEYSPACE);
-                assertEquals(6, ks.getColumnFamilyStores().size());
+                assertEquals(7, ks.getColumnFamilyStores().size());
                 for (int i = 0; i < 6; i++)
                     assertTrue(ks.getColumnFamilyStore("x"+i) != null); // getColumnFamilyStore throws
+                assertTrue(ks.getColumnFamilyStore("yy") != null);
             });
         }
     }
diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java
index fd9dbd4bc3..4d78c20e48 100644
--- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java
+++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java
@@ -219,7 +219,8 @@ public class AccordTopologyMixupTest extends TopologyMixupTestBase<AccordTopolog
         protected void onConfigure(IInstanceConfig c)
         {
             c.set("accord.shard_count", 1)
-                    .set("paxos_variant", Config.PaxosVariant.v2.name());
+             .set("accord.queue_shard_count", 1)
+             .set("paxos_variant", Config.PaxosVariant.v2.name());
         }
 
         @Override
diff --git a/test/distributed/org/apache/cassandra/service/accord/AccordJournalCompactionTest.java b/test/distributed/org/apache/cassandra/service/accord/AccordJournalCompactionTest.java
index ac17ddad15..a656111a9f 100644
--- a/test/distributed/org/apache/cassandra/service/accord/AccordJournalCompactionTest.java
+++ b/test/distributed/org/apache/cassandra/service/accord/AccordJournalCompactionTest.java
@@ -49,6 +49,7 @@ import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.journal.TestParams;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.utils.AccordGenerators;
 
 import static accord.local.CommandStores.RangesForEpoch;
@@ -111,7 +112,7 @@ public class AccordJournalCompactionTest
             {
                 return false;
             }
-        });
+        }, new AccordAgent());
         try
         {
             journal.start(null);
diff --git a/test/harry/main/org/apache/cassandra/harry/harry2/RectangleToSquares.java b/test/harry/main/org/apache/cassandra/harry/harry2/RectangleToSquares.java
new file mode 100644
index 0000000000..cf2c121c2a
--- /dev/null
+++ b/test/harry/main/org/apache/cassandra/harry/harry2/RectangleToSquares.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.harry2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RectangleToSquares {
+
+    // Class to store dimensions of sub-rectangles
+    static class SubRectangle {
+        double width;
+        double height;
+
+        SubRectangle(double width, double height) {
+            this.width = width;
+            this.height = height;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("Width: %.2f, Height: %.2f", width, height);
+        }
+    }
+
+    // Heron's method to compute square root
+    public static double heronsSquareRoot(double single_person_area, double tolerance) {
+        double x = single_person_area;
+        double prevGuess;
+        do {
+            prevGuess = x;
+            x = 0.5 * (x + single_person_area / x);  // Heron's iterative formula
+        } while (Math.abs(x - prevGuess) > tolerance);
+        return x;
+    }
+
+    // Function to divide the rectangle into near-square sub-rectangles
+    public static List<SubRectangle> divideIntoSquares(double M, double N, int number_of_people, double tolerance) {
+        List<SubRectangle> subRectangles = new ArrayList<>();
+
+        // Compute the area of each sub-rectangle
+        double totalArea = M * N;
+        double areaPerSubRectangle = totalArea / number_of_people;
+
+        // Use Heron's method to find the ideal side length for near-square sub-rectangles
+        double idealSideLength = heronsSquareRoot(areaPerSubRectangle, tolerance);
+
+        // Determine how many divisions along width and height result in near-square sub-rectangles
+        int widthDivisions = (int) (M / idealSideLength);
+        int heightDivisions = (int) (N / idealSideLength);
+
+
+
+        return subRectangles;
+    }
+
+    public static void main(String[] args) {
+        double M = 20;
+        double N = 10;
+        int numberOfSubRectangles = 6;
+        double tolerance = 0.1;
+
+        // Divide the rectangle into sub-rectangles that are near-squares
+        List<SubRectangle> subRectangles = divideIntoSquares(M, N, numberOfSubRectangles, tolerance);
+
+        // Print the dimensions of the sub-rectangles
+        for (SubRectangle subRectangle : subRectangles) {
+            System.out.println(subRectangle);
+        }
+    }
+}
diff --git a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
index 5043012472..81b4a288e9 100644
--- a/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
+++ b/test/simulator/asm/org/apache/cassandra/simulator/asm/InterceptClasses.java
@@ -70,7 +70,9 @@ public class InterceptClasses implements BiFunction<String, byte[], byte[]>
     private static final Pattern GLOBAL_METHODS = Pattern.compile("org[/.]apache[/.]cassandra[/.](?!simulator[/.]).*" +
                                                                   "|org[/.]apache[/.]cassandra[/.]simulator[/.]test[/.].*" +
                                                                   "|org[/.]apache[/.]cassandra[/.]simulator[/.]cluster[/.].*" +
-                                                                  "|io[/.]netty[/.]util[/.]concurrent[/.]FastThreadLocal"); // intercept IdentityHashMap for execution consistency
+                                                                  "|io[/.]netty[/.]util[/.]concurrent[/.]FastThreadLocal" +
+                                                                  "|accord[/.].*"
+    ); // intercept IdentityHashMap for execution consistency
     private static final Pattern NEMESIS = GLOBAL_METHODS;
     private static final Set<String> WARNED = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
diff --git a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
index ffd918584b..f07040cf6b 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/ClusterSimulation.java
@@ -779,10 +779,12 @@ public class ClusterSimulation<S extends Simulation> implements AutoCloseable
                                    .set("memtable_allocation_type", builder.memoryListener != null ? "unslabbed_heap_buffers_logged" : "heap_buffers")
                                    .set("file_cache_size", "16MiB")
                                    .set("use_deterministic_table_id", true)
+                                   .set("accord.queue_submission_model", "EXEC_ST")
                                    .set("disk_access_mode", "standard")
                                    .set("failure_detector", SimulatedFailureDetector.Instance.class.getName())
                                    .set("commitlog_compression", new ParameterizedClass(LZ4Compressor.class.getName(), emptyMap()))
-                                   .set("commitlog_sync", "batch");
+                                   .set("commitlog_sync", "batch")
+                                   .set("accord.journal.flush_mode", "BATCH");
                              // TODO: Add remove() to IInstanceConfig
                              if (config instanceof InstanceConfig)
                              {
diff --git a/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java b/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java
index 73bb343489..16233a5e8d 100644
--- a/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java
+++ b/test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.concurrent.ExecutorFactory;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.config.AccordSpec;
 import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DurationSpec;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.filesystem.ListenableFileSystem;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -75,9 +76,12 @@ public class AccordJournalSimulationTest extends SimulationTestBase
 
                      Keyspace.setInitialized();
 
+                     AccordSpec.JournalSpec spec = new AccordSpec.JournalSpec();
+                     spec.flushPeriod = new DurationSpec.IntSecondsBound(1);
+
                      State.journal = new Journal<>("AccordJournal",
                                                    new File("/journal"),
-                                                   new AccordSpec.JournalSpec(),
+                                                   spec,
                                                    new IdentityKeySerializer(),
                                                    new IdentityValueSerializer(),
                                                    SegmentCompactor.noop());
@@ -96,7 +100,7 @@ public class AccordJournalSimulationTest extends SimulationTestBase
                 int finalI = i;
                 State.executor.submit(() -> {
                     RecordPointer ptr = State.journal.asyncWrite("test" + finalI, "test" + finalI, Collections.singleton(1));
-                    State.journal.onFlush(ptr, State.latch::decrement);
+                    State.journal.onDurable(ptr, State.latch::decrement);
                 });
             }
 
@@ -189,6 +193,37 @@ public class AccordJournalSimulationTest extends SimulationTestBase
             return new String(key);
         }
 
+        @Override
+        public void serialize(String key, ByteBuffer out, int userVersion) throws IOException
+        {
+            int maxSize = 16 - TypeSizes.INT_SIZE;
+            if (key.length() > maxSize)
+                throw new IllegalStateException();
+
+            out.putInt(key.length());
+            for (int i = 0 ; i < key.length() ; ++i)
+                out.put((byte)key.charAt(i));
+            int remaining = maxSize - key.length();
+            for (int i = 0; i < remaining; i++)
+                out.put((byte) (aByte + i));
+        }
+
+        @Override
+        public String deserialize(ByteBuffer in, int userVersion)
+        {
+            int size = in.getInt();
+            byte[] key = new byte[size];
+            for (int i = 0; i < size; i++)
+                key[i] = in.get();
+
+            int maxSize = 16 - TypeSizes.INT_SIZE;
+            int remaining = maxSize - size;
+            for (int i = 0; i < remaining; i++)
+                Assert.assertEquals(aByte + i, in.get());
+
+            return new String(key);
+        }
+
         @Override
         public String deserialize(ByteBuffer buffer, int position, int userVersion)
         {
diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java
index 08b91075fa..88bdc7f360 100644
--- a/test/unit/org/apache/cassandra/ServerTestUtils.java
+++ b/test/unit/org/apache/cassandra/ServerTestUtils.java
@@ -32,6 +32,7 @@ import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import accord.impl.basic.SimulatedFault;
 import org.apache.cassandra.audit.AuditLogManager;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -187,7 +188,8 @@ public final class ServerTestUtils
         {
             public void uncaughtException(Thread t, Throwable e)
             {
-                logger.error("Fatal exception in thread " + t, e);
+                if (e instanceof SimulatedFault) logger.error("SimulatedFault {} in thread {}", e.getMessage(), t);
+                else logger.error("Fatal exception in thread " + t, e);
             }
         });
 
diff --git a/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java
index d75c10eaa9..cfa8d6f449 100644
--- a/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java
+++ b/test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java
@@ -33,7 +33,7 @@ import org.apache.cassandra.utils.WithResources;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
 
-public class ForwardingExecutorPlus implements ExecutorPlus, SequentialExecutorPlus
+public class ForwardingExecutorPlus implements ExecutorPlus, SequentialExecutorPlus, LocalAwareExecutorPlus
 {
     private final ExecutorService delegate;
 
diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
index 2aa0fe2c83..5712a7fff1 100644
--- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
+++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java
@@ -81,6 +81,8 @@ public class DatabaseDescriptorRefTest
     "org.apache.cassandra.config.AccordSpec$JournalSpec",
     "org.apache.cassandra.config.AccordSpec$MinEpochRetrySpec",
     "org.apache.cassandra.config.AccordSpec$TransactionalRangeMigration",
+    "org.apache.cassandra.config.AccordSpec$QueueShardModel",
+    "org.apache.cassandra.config.AccordSpec$QueueSubmissionModel",
     "org.apache.cassandra.config.CassandraRelevantProperties",
     "org.apache.cassandra.config.CassandraRelevantProperties$PropertyConverter",
     "org.apache.cassandra.config.Config",
diff --git a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
index 6a4546f435..85f5d94832 100644
--- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
+++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
@@ -437,11 +437,13 @@ public class YamlConfigurationLoaderTest
         {
             Config c = fromType(type, "available_processors", 4);
             assertThat(c.available_processors).isEqualTo(new OptionaldPositiveInt(4));
-            assertThat(c.accord.shard_count).isEqualTo(OptionaldPositiveInt.UNDEFINED);
+            assertThat(c.accord.command_store_shard_count).isEqualTo(OptionaldPositiveInt.UNDEFINED);
+            assertThat(c.accord.queue_shard_count).isEqualTo(OptionaldPositiveInt.UNDEFINED);
 
-            c = fromType(type, "available_processors", 3, "accord.shard_count", 1);
+            c = fromType(type, "available_processors", 3, "accord.queue_shard_count", 1, "accord.command_store_shard_count", 1);
             assertThat(c.available_processors).isEqualTo(new OptionaldPositiveInt(3));
-            assertThat(c.accord.shard_count).isEqualTo(new OptionaldPositiveInt(1));
+            assertThat(c.accord.command_store_shard_count).isEqualTo(new OptionaldPositiveInt(1));
+            assertThat(c.accord.queue_shard_count).isEqualTo(new OptionaldPositiveInt(1));
         }
     }
 
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index c9e0ced979..98a8ef29c8 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -180,7 +180,7 @@ import org.apache.cassandra.serializers.TypeSerializer;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordStateCache;
+import org.apache.cassandra.service.accord.AccordCache;
 import org.apache.cassandra.service.snapshot.SnapshotManager;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.transport.Event;
@@ -390,7 +390,7 @@ public abstract class CQLTester
     public static void prepareServer()
     {
         ServerTestUtils.prepareServer();
-        AccordStateCache.validateLoadOnEvict(true);
+        AccordCache.validateLoadOnEvict(true);
     }
 
     public static void cleanup()
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 0febbe660c..c134eaed43 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -842,7 +842,7 @@ public class ColumnFamilyStoreTest
         {
 
             @Override
-            public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup)
+            public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup, boolean assumeMissing)
             {
                 return 0;
             }
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java
index ca59f202b7..da4693a847 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java
@@ -24,7 +24,9 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.LockSupport;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
@@ -93,7 +95,7 @@ import org.apache.cassandra.utils.Pair;
 import org.assertj.core.api.Assertions;
 
 import static accord.impl.TimestampsForKey.NO_LAST_EXECUTED_HLC;
-import static accord.local.KeyHistory.COMMANDS;
+import static accord.local.KeyHistory.SYNC;
 import static accord.local.PreLoadContext.contextFor;
 import static accord.primitives.Routable.Domain.Range;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
@@ -385,7 +387,7 @@ public class CompactionAccordIteratorsTest
     {
         Ranges ranges = AccordTestUtils.fullRange(AccordTestUtils.keys(table, 42));
         txnId = txnId.as(Kind.Read, Range);
-        return RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, txnId, txnId, txnId, LT_TXN_ID.as(Range));
+        return RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, txnId, txnId, txnId, txnId, LT_TXN_ID.as(Range));
     }
 
     enum DurableBeforeType
@@ -439,14 +441,18 @@ public class CompactionAccordIteratorsTest
     {
         commandStore.executeBlocking(() -> {
             // clear cache and wait for post-eviction writes to complete
-            long cacheSize = commandStore.cache().capacity();
-            commandStore.cache().setCapacity(0);
-            commandStore.cache().setCapacity(cacheSize);
-            commandStore.cache().awaitSaveResults();
+            try (AccordExecutor.ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();)
+            {
+                long cacheSize = cache.global.capacity();
+                cache.global.setCapacity(0);
+                cache.global.setCapacity(cacheSize);
+            }
         });
         commands.forceBlockingFlush(FlushReason.UNIT_TESTS);
         timestampsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
         commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
+        while (commandStore.executor().hasTasks())
+            LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100));
     }
 
     private void testWithCommandStore(TestWithCommandStore test, boolean additionalCommand) throws Throwable
@@ -473,19 +479,19 @@ public class CompactionAccordIteratorsTest
             PartialDeps partialDeps = Deps.NONE.intersecting(AccordTestUtils.fullRange(txn));
             PartialTxn partialTxn = txn.slice(commandStore.unsafeRangesForEpoch().currentRanges(), true);
             Route<?> partialRoute = route.slice(commandStore.unsafeRangesForEpoch().currentRanges());
-            getUninterruptibly(commandStore.execute(contextFor(txnId, route, COMMANDS), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, route, SYNC), safe -> {
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToKeyspace(commandStore));
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, route, COMMANDS), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, route, SYNC), safe -> {
                 CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, txnId, partialDeps, appendDiffToKeyspace(commandStore));
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, route, COMMANDS), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, route, SYNC), safe -> {
                 CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, txnId, partialDeps, appendDiffToKeyspace(commandStore));
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, route, COMMANDS), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, route, SYNC), safe -> {
                 Pair<Writes, Result> result = AccordTestUtils.processTxnResultDirect(safe, txnId, partialTxn, txnId);
                 CheckedCommands.apply(safe, txnId, route, txnId, partialDeps, partialTxn, result.left, result.right, appendDiffToKeyspace(commandStore));
             }).beginAsResult());
@@ -493,7 +499,7 @@ public class CompactionAccordIteratorsTest
             // The apply chain is asychronous, so it is easiest to just spin until it is applied
             // in order to have the updated state in the system table
             spinAssertEquals(true, 5, () -> {
-                return getUninterruptibly(commandStore.submit(contextFor(txnId, route, COMMANDS), safe -> {
+                return getUninterruptibly(commandStore.submit(contextFor(txnId, route, SYNC), safe -> {
                     StoreParticipants participants = StoreParticipants.all(route);
                     Command command = safe.get(txnId, participants).current();
                     appendDiffToKeyspace(commandStore).accept(null, command);
diff --git a/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java b/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java
index 7277edfa15..d9bdc1d75a 100644
--- a/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java
@@ -63,7 +63,8 @@ public class AccordVirtualTablesTest extends CQLTester
     public static void setUpClass()
     {
         daemonInitialization();
-        DatabaseDescriptor.getAccord().shard_count = new OptionaldPositiveInt(1);
+        DatabaseDescriptor.getAccord().queue_shard_count = new OptionaldPositiveInt(1);
+        DatabaseDescriptor.getAccord().command_store_shard_count = new OptionaldPositiveInt(1);
 
         CQLTester.setUpClass();
 
@@ -215,6 +216,9 @@ public class AccordVirtualTablesTest extends CQLTester
                 case ACCORD_CHECK_STATUS_REQ:
                 case ACCORD_CHECK_STATUS_RSP:
                 case ACCORD_READ_RSP:
+                case ACCORD_AWAIT_REQ:
+                case ACCORD_AWAIT_RSP:
+                case ACCORD_AWAIT_ASYNC_RSP_REQ:
                     return true;
                 default:
                     // many code paths don't log the error...
diff --git a/test/unit/org/apache/cassandra/index/accord/AccordIndexStressTest.java b/test/unit/org/apache/cassandra/index/accord/AccordIndexStressTest.java
index bf41340c88..f660d4e1c2 100644
--- a/test/unit/org/apache/cassandra/index/accord/AccordIndexStressTest.java
+++ b/test/unit/org/apache/cassandra/index/accord/AccordIndexStressTest.java
@@ -352,7 +352,9 @@ public class AccordIndexStressTest extends CQLTester
 
     private Set<TxnId> readIndex(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId)
     {
-        return searcher.intersects(store, start, end, minTxnId, maxTxnId);
+        Set<TxnId> out = new ObjectHashSet<>();
+        searcher.intersects(store, start, end, minTxnId, maxTxnId, out::add);
+        return out;
     }
 
     private Set<TxnId> readCQL(int store, AccordRoutingKey start, AccordRoutingKey end, TxnId minTxnId, Timestamp maxTxnId)
diff --git a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
index 3716236756..fe2a515ca1 100644
--- a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
@@ -54,6 +54,7 @@ import accord.utils.Property.UnitCommand;
 import accord.utils.RandomSource;
 import org.agrona.collections.Int2ObjectHashMap;
 import org.agrona.collections.Long2ObjectHashMap;
+import org.agrona.collections.ObjectHashSet;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
@@ -346,7 +347,9 @@ public class RouteIndexTest extends CQLTester.InMemory
         @Override
         public Set<TxnId> run(ColumnFamilyStore sut) throws Throwable
         {
-            return ROUTES_SEARCHER.intersects(storeId, range, TxnId.NONE, Timestamp.MAX);
+            Set<TxnId> out = new ObjectHashSet<>();
+            ROUTES_SEARCHER.intersects(storeId, range, TxnId.NONE, Timestamp.MAX, out::add);
+            return out;
         }
 
         @Override
diff --git a/test/unit/org/apache/cassandra/journal/SegmentTest.java b/test/unit/org/apache/cassandra/journal/SegmentTest.java
index d78fae8ecf..8700c805fa 100644
--- a/test/unit/org/apache/cassandra/journal/SegmentTest.java
+++ b/test/unit/org/apache/cassandra/journal/SegmentTest.java
@@ -24,10 +24,12 @@ import java.util.*;
 
 import org.junit.Test;
 
+import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
 import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 import static org.junit.Assert.assertEquals;
@@ -136,7 +138,7 @@ public class SegmentTest
         activeSegment.allocate(record3.remaining(), hosts3).write(id3, record3, hosts3);
         activeSegment.allocate(record4.remaining(), hosts4).write(id4, record4, hosts4);
 
-        activeSegment.close();
+        activeSegment.close(null);
 
         StaticSegment<TimeUUID, ByteBuffer> staticSegment = StaticSegment.open(descriptor, TimeUUIDKeySupport.INSTANCE);
 
@@ -201,7 +203,12 @@ public class SegmentTest
         activeSegment.allocate(record3.remaining(), hosts3).write(id3, record3, hosts3);
         activeSegment.allocate(record4.remaining(), hosts4).write(id4, record4, hosts4);
 
-        activeSegment.close();
+        Segment.Tidier tidier = (Segment.Tidier)activeSegment.selfRef().tidier();
+        tidier.executor = ImmediateExecutor.INSTANCE;
+        OpOrder opOrder = new OpOrder();
+        tidier.await = opOrder.newBarrier();
+        tidier.await.issue();
+        activeSegment.close(null);
 
         StaticSegment.SequentialReader<TimeUUID> reader = StaticSegment.sequentialReader(descriptor, TimeUUIDKeySupport.INSTANCE, 0);
 
diff --git a/test/unit/org/apache/cassandra/journal/SyncedOffsetsTest.java b/test/unit/org/apache/cassandra/journal/SyncedOffsetsTest.java
deleted file mode 100644
index b5df2b6b22..0000000000
--- a/test/unit/org/apache/cassandra/journal/SyncedOffsetsTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.journal;
-
-import java.io.IOException;
-import java.nio.file.Files;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.io.util.File;
-
-import static org.junit.Assert.assertEquals;
-
-public class SyncedOffsetsTest
-{
-    @BeforeClass
-    public static void setUp()
-    {
-        DatabaseDescriptor.clientInitialization();
-    }
-
-    @Test
-    public void testCommonCase() throws IOException
-    {
-        testReadWrite(512, true);
-        testReadWrite(512, false);
-    }
-
-    @Test
-    public void testResize() throws IOException
-    {
-        testReadWrite(2048, true);
-        testReadWrite(2048, false);
-    }
-
-    private void testReadWrite(int n, boolean syncOnMark) throws IOException
-    {
-        File directory = new File(Files.createTempDirectory(null));
-        directory.deleteOnExit();
-
-        Descriptor descriptor = Descriptor.create(directory, System.currentTimeMillis(), 1);
-
-        SyncedOffsets active = SyncedOffsets.active(descriptor);
-        for (int i = 0; i < n; i++)
-            active.mark(i, syncOnMark);
-        assertEquals(n - 1, active.syncedOffset());
-        active.close();
-
-        SyncedOffsets loaded = SyncedOffsets.load(descriptor);
-        assertEquals(n - 1, loaded.syncedOffset());
-        loaded.close();
-    }
-}
diff --git a/test/unit/org/apache/cassandra/journal/TestParams.java b/test/unit/org/apache/cassandra/journal/TestParams.java
index 5773c4763a..0cab7bcd6d 100644
--- a/test/unit/org/apache/cassandra/journal/TestParams.java
+++ b/test/unit/org/apache/cassandra/journal/TestParams.java
@@ -17,6 +17,8 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.concurrent.TimeUnit;
+
 import org.apache.cassandra.net.MessagingService;
 
 public class TestParams implements Params
@@ -48,21 +50,21 @@ public class TestParams implements Params
     }
 
     @Override
-    public int compactionPeriodMillis()
+    public long compactionPeriod(TimeUnit units)
     {
-        return 60_000;
+        return units.convert(60, TimeUnit.SECONDS);
     }
 
     @Override
-    public int flushPeriodMillis()
+    public long flushPeriod(TimeUnit units)
     {
-        return 1000;
+        return units.convert(1, TimeUnit.SECONDS);
     }
 
     @Override
-    public int periodicFlushLagBlock()
+    public long periodicBlockPeriod(TimeUnit units)
     {
-        return 1500;
+        return units.convert(2, TimeUnit.SECONDS);
     }
 
     @Override
diff --git a/test/unit/org/apache/cassandra/journal/TimeUUIDKeySupport.java b/test/unit/org/apache/cassandra/journal/TimeUUIDKeySupport.java
index 3d04fad89b..5694a29e7a 100644
--- a/test/unit/org/apache/cassandra/journal/TimeUUIDKeySupport.java
+++ b/test/unit/org/apache/cassandra/journal/TimeUUIDKeySupport.java
@@ -44,6 +44,13 @@ class TimeUUIDKeySupport implements KeySupport<TimeUUID>
         out.writeLong(key.lsb());
     }
 
+    @Override
+    public void serialize(TimeUUID key, ByteBuffer out, int userVersion) throws IOException
+    {
+        out.putLong(key.uuidTimestamp());
+        out.putLong(key.lsb());
+    }
+
     @Override
     public TimeUUID deserialize(DataInputPlus in, int userVersion) throws IOException
     {
@@ -60,6 +67,14 @@ class TimeUUIDKeySupport implements KeySupport<TimeUUID>
         return new TimeUUID(uuidTimestamp, lsb);
     }
 
+    @Override
+    public TimeUUID deserialize(ByteBuffer buffer, int userVersion)
+    {
+        long uuidTimestamp = buffer.getLong();
+        long lsb = buffer.getLong();
+        return new TimeUUID(uuidTimestamp, lsb);
+    }
+
     @Override
     public void updateChecksum(Checksum crc, TimeUUID key, int userVersion)
     {
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java
new file mode 100644
index 0000000000..0cd0d4e031
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheEntryTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
+import org.apache.cassandra.service.accord.AccordCache.Type;
+
+public class AccordCacheEntryTest
+{
+    static class CacheEntry extends AccordCacheEntry<String, String>
+    {
+        public CacheEntry(String key, Type<String, String, ?>.Instance instance)
+        {
+            super(key, instance);
+        }
+
+        public CacheEntry(String key)
+        {
+            this(key, null);
+        }
+    }
+
+    private static void assertIllegalState(Runnable runnable)
+    {
+        try
+        {
+            runnable.run();
+            Assert.fail("Expected IllegalStateException");
+        }
+        catch (IllegalStateException ise)
+        {
+            // expected
+        }
+    }
+
+    @Test
+    public void loadSuccessTest()
+    {
+        CacheEntry state = new CacheEntry("K");
+
+        Assert.assertEquals(Status.UNINITIALIZED, state.status());
+        assertIllegalState(state::getExclusive);
+        assertIllegalState(() -> state.setExclusive("VVVV"));
+        assertIllegalState(state::loading);
+
+        state.readyToLoad();
+        state.testLoad();
+        Assert.assertEquals(Status.LOADING, state.status());
+
+        state.testLoaded("V");
+        Assert.assertEquals(Status.LOADED, state.status());
+        Assert.assertEquals("V", state.getExclusive());
+
+        assertIllegalState(state::testLoad);
+        assertIllegalState(() -> state.loaded(null));
+        assertIllegalState(state::loading);
+    }
+
+    @Test
+    public void loadNullTest()
+    {
+        CacheEntry state = new CacheEntry("K");
+        Assert.assertEquals(Status.UNINITIALIZED, state.status());
+
+        assertIllegalState(state::getExclusive);
+        assertIllegalState(() -> state.setExclusive("VVVV"));
+        assertIllegalState(state::loading);
+
+        state.readyToLoad();
+        state.testLoad();
+        Assert.assertEquals(Status.LOADING, state.status());
+
+        // TODO (expected): this is sort of a pointless test now - remove it?
+        state.testLoaded(null);
+        Assert.assertEquals(Status.LOADED, state.status());
+        Assert.assertNull(state.getExclusive());
+
+        assertIllegalState(state::testLoad);
+        assertIllegalState(state::failedToLoad);
+        assertIllegalState(state::loading);
+    }
+
+    @Test
+    public void loadFailureTest()
+    {
+        CacheEntry state = new CacheEntry("K");
+
+        Assert.assertEquals(Status.UNINITIALIZED, state.status());
+        assertIllegalState(state::getExclusive);
+        assertIllegalState(() -> state.setExclusive("VVVV"));
+        assertIllegalState(state::loading);
+
+        state.readyToLoad();
+        state.testLoad();
+        state.failedToLoad();
+        Assert.assertEquals(Status.FAILED_TO_LOAD, state.status());
+        assertIllegalState(state::getExclusive);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordStateCacheTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java
similarity index 54%
rename from test/unit/org/apache/cassandra/service/accord/AccordStateCacheTest.java
rename to test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java
index 50dd331e8f..d7a89017d6 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordStateCacheTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordCacheTest.java
@@ -18,17 +18,20 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.UUID;
+import java.util.function.Function;
 
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import accord.utils.async.AsyncChain;
+import accord.utils.async.Cancellable;
 import org.apache.cassandra.cache.CacheSize;
+import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.ManualExecutor;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
 import org.apache.cassandra.metrics.CacheAccessMetrics;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
+import org.apache.cassandra.service.accord.AccordCacheEntry.OnSaved;
+import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
 
 import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -36,23 +39,23 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class AccordStateCacheTest
+public class AccordCacheTest
 {
     private static final long DEFAULT_NODE_SIZE = nodeSize(0);
-    private AccordStateCacheMetrics cacheMetrics;
+    private AccordCacheMetrics cacheMetrics;
 
     private static abstract class TestSafeState<T> implements AccordSafeState<T, T>
     {
         protected boolean invalidated = false;
-        protected final AccordCachingState<T, T> global;
+        protected final AccordCacheEntry<T, T> global;
         private T original = null;
 
-        public TestSafeState(AccordCachingState<T, T> global)
+        public TestSafeState(AccordCacheEntry<T, T> global)
         {
             this.global = global;
         }
 
-        public AccordCachingState<T, T> global()
+        public AccordCacheEntry<T, T> global()
         {
             return global;
         }
@@ -66,13 +69,13 @@ public class AccordStateCacheTest
         @Override
         public T current()
         {
-            return global.get();
+            return global.getExclusive();
         }
 
         @Override
         public void set(T update)
         {
-            global.set(update);
+            global.setExclusive(update);
         }
 
         @Override
@@ -84,23 +87,11 @@ public class AccordStateCacheTest
         @Override
         public void preExecute()
         {
-            original = global.get();
+            original = global.getExclusive();
         }
 
         @Override
-        public Status globalStatus()
-        {
-            return global.status();
-        }
-
-        @Override
-        public AsyncChain<?> loading()
-        {
-            return global.loading();
-        }
-
-        @Override
-        public AsyncChain<?> saving()
+        public Cancellable saving()
         {
             return global.saving();
         }
@@ -126,7 +117,7 @@ public class AccordStateCacheTest
 
     private static class SafeString extends TestSafeState<String>
     {
-        public SafeString(AccordCachingState<String, String> global)
+        public SafeString(AccordCacheEntry<String, String> global)
         {
             super(global);
         }
@@ -134,7 +125,7 @@ public class AccordStateCacheTest
 
     private static class SafeInt extends TestSafeState<Integer>
     {
-        public SafeInt(AccordCachingState<Integer, Integer> global)
+        public SafeInt(AccordCacheEntry<Integer, Integer> global)
         {
             super(global);
         }
@@ -142,7 +133,7 @@ public class AccordStateCacheTest
 
     private static long emptyNodeSize()
     {
-        return AccordCachingState.EMPTY_SIZE;
+        return AccordCacheEntry.EMPTY_SIZE;
     }
 
     private static long nodeSize(long itemSize)
@@ -150,22 +141,22 @@ public class AccordStateCacheTest
         return itemSize + emptyNodeSize();
     }
 
-    private static void assertCacheState(AccordStateCache cache, int referenced, int total, long bytes)
+    private static void assertCacheState(AccordCache cache, int referenced, int total, long bytes)
     {
         Assert.assertEquals(referenced, cache.numReferencedEntries());
         Assert.assertEquals(total, cache.size());
         Assert.assertEquals(bytes, cache.weightedSize());
     }
 
-    private void assertCacheMetrics(CacheAccessMetrics metrics, int hits, int misses, int requests)
+    private void assertCacheMetrics(CacheAccessMetrics metrics, int hits, int misses, int requests, int sizes)
     {
         Assert.assertEquals(hits, metrics.hits.getCount());
         Assert.assertEquals(misses, metrics.misses.getCount());
         Assert.assertEquals(requests, metrics.requests.getCount());
-        if (metrics instanceof AccordStateCacheMetrics)
+        if (metrics instanceof AccordCacheMetrics)
         {
-            AccordStateCacheMetrics ascMetrics = (AccordStateCacheMetrics) metrics;
-            Assert.assertEquals(misses, ascMetrics.objectSize.getCount());
+            AccordCacheMetrics ascMetrics = (AccordCacheMetrics) metrics;
+            Assert.assertEquals(sizes, ascMetrics.objectSize.getCount());
             assertThat(ascMetrics.objectSize.getSnapshot().getMax()).isGreaterThanOrEqualTo(DEFAULT_NODE_SIZE);
         }
     }
@@ -174,89 +165,93 @@ public class AccordStateCacheTest
     public void before()
     {
         String type = String.format("%s-%s", AccordCommandStores.ACCORD_STATE_CACHE, UUID.randomUUID());
-        cacheMetrics = new AccordStateCacheMetrics(type);
+        cacheMetrics = new AccordCacheMetrics(type);
     }
-
+    
     @Test
     public void testAcquisitionAndRelease()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), 500, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+            cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString safeString1 = instance.acquire("1");
         assertCacheState(cache, 1, 1, emptyNodeSize());
-        testLoad(executor, safeString1, "1");
-        Assert.assertTrue(cache.isEmpty());
+        testLoad(executor, instance, safeString1, "1");
+        Assert.assertTrue(!cache.evictionQueue().iterator().hasNext());
 
-        instance.release(safeString1);
+        instance.release(safeString1, null);
         assertCacheState(cache, 0, 1, nodeSize(1));
         Assert.assertSame(safeString1.global, cache.head());
         Assert.assertSame(safeString1.global, cache.tail());
 
         SafeString safeString2 = instance.acquire("2");
         assertCacheState(cache, 1, 2, DEFAULT_NODE_SIZE + nodeSize(1));
-        testLoad(executor, safeString2, "2");
-        instance.release(safeString2);
+        testLoad(executor, instance, safeString2, "2");
+        instance.release(safeString2, null);
         assertCacheState(cache, 0, 2, nodeSize(1) + nodeSize(1));
 
         Assert.assertSame(safeString1.global, cache.head());
         Assert.assertSame(safeString2.global, cache.tail());
 
-        assertCacheMetrics(cache.metrics, 0, 2, 2);
-        assertCacheMetrics(instance.instanceMetrics, 0, 2, 2);
+        assertCacheMetrics(cache.metrics, 0, 2, 2, 2);
+        assertCacheMetrics(type.typeMetrics, 0, 2, 2, 2);
     }
 
     @Test
     public void testCachingMetricsWithTwoInstances()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> stringInstance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true,String::length);
-        AccordStateCache.Instance<Integer, Integer, SafeInt> intInstance =
-            cache.instance(Integer.class, SafeInt.class, SafeInt::new, key -> key, (current) -> null, (k, v) -> true,ignored -> Integer.BYTES);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), 500, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> stringType =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance stringInstance = stringType.newInstance(null);
+        AccordCache.Type<Integer, Integer, SafeInt> intType =
+        cache.newType(Integer.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, ignore -> Integer.BYTES, SafeInt::new);
         assertCacheState(cache, 0, 0, 0);
+        AccordCache.Type<Integer, Integer, SafeInt>.Instance intInstance = intType.newInstance(null);
 
         SafeString safeString1 = stringInstance.acquire("1");
-        testLoad(executor, safeString1, "1");
-        stringInstance.release(safeString1);
+        testLoad(executor, stringInstance, safeString1, "1");
+        stringInstance.release(safeString1, null);
         SafeString safeString2 = stringInstance.acquire("2");
-        testLoad(executor, safeString2, "2");
-        stringInstance.release(safeString2);
+        testLoad(executor, stringInstance, safeString2, "2");
+        stringInstance.release(safeString2, null);
 
         SafeInt safeInt1 = intInstance.acquire(3);
-        testLoad(executor, safeInt1, 3);
-        intInstance.release(safeInt1);
+        testLoad(executor, intInstance, safeInt1, 3);
+        intInstance.release(safeInt1, null);
         SafeInt safeInt2 = intInstance.acquire(4);
-        testLoad(executor, safeInt2, 4);
-        intInstance.release(safeInt2);
+        testLoad(executor, intInstance, safeInt2, 4);
+        intInstance.release(safeInt2, null);
         SafeInt safeInt3 = intInstance.acquire(5);
-        testLoad(executor, safeInt3, 5);
-        intInstance.release(safeInt3);
+        testLoad(executor, intInstance, safeInt3, 5);
+        intInstance.release(safeInt3, null);
 
         assertCacheState(cache, 0, 5, nodeSize(Integer.BYTES) * 3 + nodeSize(1) * 2);
-        assertThat(stringInstance.size()).isEqualTo(2);
-        assertThat(stringInstance.weightedSize()).isEqualTo(nodeSize(1) * 2);
-        assertThat(stringInstance.capacity()).isEqualTo(cache.capacity());
-        assertThat(intInstance.size()).isEqualTo(3);
-        assertThat(intInstance.weightedSize()).isEqualTo(nodeSize(Integer.BYTES) * 3);
-        assertThat(intInstance.capacity()).isEqualTo(cache.capacity());
+        assertThat(stringType.size()).isEqualTo(2);
+        assertThat(stringType.weightedSize()).isEqualTo(nodeSize(1) * 2);
+        assertThat(stringType.capacity()).isEqualTo(cache.capacity());
+        assertThat(intType.size()).isEqualTo(3);
+        assertThat(intType.weightedSize()).isEqualTo(nodeSize(Integer.BYTES) * 3);
+        assertThat(intType.capacity()).isEqualTo(cache.capacity());
 
-        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> stringInstance.setCapacity(123));
-        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> intInstance.setCapacity(123));
+        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> stringType.setCapacity(123));
+        assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> intType.setCapacity(123));
     }
 
     @Test
     public void testRotation()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, DEFAULT_NODE_SIZE * 5, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), DEFAULT_NODE_SIZE * 5, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
         assertCacheState(cache, 0, 0, 0);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
 
         SafeString[] items = new SafeString[3];
         for (int i=0; i<3; i++)
@@ -264,26 +259,26 @@ public class AccordStateCacheTest
             SafeString safeString = instance.acquire(Integer.toString(i));
             items[i] = safeString;
             Assert.assertNotNull(safeString);
-            testLoad(executor, safeString, Integer.toString(i));
+            testLoad(executor, instance, safeString, Integer.toString(i));
             Assert.assertTrue(instance.isReferenced(safeString.key()));
-            instance.release(safeString);
+            instance.release(safeString, null);
         }
 
         Assert.assertSame(items[0].global, cache.head());
         Assert.assertSame(items[2].global, cache.tail());
         assertCacheState(cache, 0, 3, nodeSize(1) * 3);
-        assertCacheMetrics(cache.metrics, 0, 3, 3);
-        assertCacheMetrics(instance.instanceMetrics, 0, 3, 3);
+        assertCacheMetrics(cache.metrics, 0, 3, 3, 3);
+        assertCacheMetrics(type.typeMetrics, 0, 3, 3, 3);
 
         SafeString safeString = instance.acquire("1");
-        Assert.assertEquals(Status.LOADED, safeString.globalStatus());
+        Assert.assertEquals(Status.LOADED, safeString.global.status());
 
         assertCacheState(cache, 1, 3, nodeSize(1) * 3);
-        assertCacheMetrics(cache.metrics, 1, 3, 4);
-        assertCacheMetrics(instance.instanceMetrics, 1, 3, 4);
+        assertCacheMetrics(cache.metrics, 1, 3, 4, 3);
+        assertCacheMetrics(type.typeMetrics, 1, 3, 4, 3);
 
         // releasing item should return it to the tail
-        instance.release(safeString);
+        instance.release(safeString, null);
         assertCacheState(cache, 0, 3, nodeSize(1) * 3);
         Assert.assertSame(items[0].global, cache.head());
         Assert.assertSame(items[1].global, cache.tail());
@@ -293,9 +288,10 @@ public class AccordStateCacheTest
     public void testEvictionOnAcquire()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 5, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), nodeSize(1) * 5, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString[] items = new SafeString[5];
@@ -303,16 +299,16 @@ public class AccordStateCacheTest
         {
             SafeString safeString = instance.acquire(Integer.toString(i));
             items[i] = safeString;
-            testLoad(executor, safeString, Integer.toString(i));
+            testLoad(executor, instance, safeString, Integer.toString(i));
             Assert.assertTrue(instance.isReferenced(safeString.key()));
-            instance.release(safeString);
+            instance.release(safeString, null);
         }
 
         assertCacheState(cache, 0, 5, nodeSize(1) * 5);
         Assert.assertSame(items[0].global, cache.head());
         Assert.assertSame(items[4].global, cache.tail());
-        assertCacheMetrics(cache.metrics, 0, 5, 5);
-        assertCacheMetrics(instance.instanceMetrics, 0, 5, 5);
+        assertCacheMetrics(cache.metrics, 0, 5, 5, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 5, 5, 5);
 
         SafeString safeString = instance.acquire("5");
         Assert.assertTrue(instance.isReferenced(safeString.key()));
@@ -323,25 +319,26 @@ public class AccordStateCacheTest
         Assert.assertSame(items[4].global, cache.tail());
         Assert.assertFalse(instance.keyIsCached("0", SafeString.class));
         Assert.assertFalse(instance.keyIsReferenced("0", SafeString.class));
-        assertCacheMetrics(cache.metrics, 0, 6, 6);
-        assertCacheMetrics(instance.instanceMetrics, 0, 6, 6);
+        assertCacheMetrics(cache.metrics, 0, 6, 6, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 6, 6, 5);
 
-        testLoad(executor, safeString, "5");
-        instance.release(safeString);
+        testLoad(executor, instance, safeString, "5");
+        instance.release(safeString, null);
         assertCacheState(cache, 0, 5, nodeSize(1) * 5);
         Assert.assertSame(items[1].global, cache.head());
         Assert.assertSame(safeString.global, cache.tail());
-        assertCacheMetrics(cache.metrics, 0, 6, 6);
-        assertCacheMetrics(instance.instanceMetrics, 0, 6, 6);
+        assertCacheMetrics(cache.metrics, 0, 6, 6, 6);
+        assertCacheMetrics(type.typeMetrics, 0, 6, 6, 6);
     }
 
     @Test
     public void testEvictionOnRelease()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 4, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), nodeSize(1) * 4, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString[] items = new SafeString[5];
@@ -349,27 +346,27 @@ public class AccordStateCacheTest
         {
             SafeString safeString = instance.acquire(Integer.toString(i));
             items[i] = safeString;
-            testLoad(executor, safeString, Integer.toString(i));
+            testLoad(executor, instance, safeString, Integer.toString(i));
             Assert.assertTrue(instance.isReferenced(safeString.key()));
         }
 
-        assertCacheState(cache, 5, 5, nodeSize(0) * 5);
-        assertCacheMetrics(cache.metrics, 0, 5, 5);
-        assertCacheMetrics(instance.instanceMetrics, 0, 5, 5);
+        assertCacheState(cache, 5, 5, nodeSize(1) * 5);
+        assertCacheMetrics(cache.metrics, 0, 5, 5, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 5, 5, 5);
         Assert.assertNull(cache.head());
         Assert.assertNull(cache.tail());
 
-        instance.release(items[2]);
-        assertCacheState(cache, 4, 4, nodeSize(0) * 4);
-        assertCacheMetrics(cache.metrics, 0, 5, 5);
-        assertCacheMetrics(instance.instanceMetrics, 0, 5, 5);
+        instance.release(items[2], null);
+        assertCacheState(cache, 4, 4, nodeSize(1) * 4);
+        assertCacheMetrics(cache.metrics, 0, 5, 5, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 5, 5, 5);
         Assert.assertNull(cache.head());
         Assert.assertNull(cache.tail());
 
-        instance.release(items[4]);
-        assertCacheState(cache, 3, 4, nodeSize(0) * 3 + nodeSize(1));
-        assertCacheMetrics(cache.metrics, 0, 5, 5);
-        assertCacheMetrics(instance.instanceMetrics, 0, 5, 5);
+        instance.release(items[4], null);
+        assertCacheState(cache, 3, 4, nodeSize(1) * 4);
+        assertCacheMetrics(cache.metrics, 0, 5, 5, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 5, 5, 5);
         Assert.assertSame(items[4].global, cache.head());
         Assert.assertSame(items[4].global, cache.tail());
     }
@@ -378,30 +375,32 @@ public class AccordStateCacheTest
     public void testMultiAcquireRelease()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, DEFAULT_NODE_SIZE * 4, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), DEFAULT_NODE_SIZE * 4, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString safeString1 = instance.acquire("0");
-        testLoad(executor, safeString1, "0");
-        Assert.assertEquals(Status.LOADED, safeString1.globalStatus());
-        assertCacheMetrics(cache.metrics, 0, 1, 1);
-        assertCacheMetrics(instance.instanceMetrics, 0, 1, 1);
+        testLoad(executor, instance, safeString1, "0");
+        Assert.assertEquals(Status.LOADED, safeString1.global.status());
+        assertCacheMetrics(cache.metrics, 0, 1, 1, 1);
+        assertCacheMetrics(type.typeMetrics, 0, 1, 1, 1);
 
         Assert.assertEquals(1, instance.references("0", SafeString.class));
-        assertCacheState(cache, 1, 1, nodeSize(0));
+        assertCacheState(cache, 1, 1, nodeSize(1));
 
         SafeString safeString2 = instance.acquire("0");
-        Assert.assertEquals(Status.LOADED, safeString1.globalStatus());
+        Assert.assertEquals("0", safeString2.current());
+        Assert.assertEquals(Status.LOADED, safeString1.global.status());
         Assert.assertEquals(2, instance.references("0", SafeString.class));
-        assertCacheState(cache, 1, 1, nodeSize(0));
-        assertCacheMetrics(cache.metrics, 1, 1, 2);
-        assertCacheMetrics(instance.instanceMetrics, 1, 1, 2);
-
-        instance.release(safeString1);
         assertCacheState(cache, 1, 1, nodeSize(1));
-        instance.release(safeString2);
+        assertCacheMetrics(cache.metrics, 1, 1, 2, 1);
+        assertCacheMetrics(type.typeMetrics, 1, 1, 2, 1);
+
+        instance.release(safeString1, null);
+        assertCacheState(cache, 1, 1, nodeSize(1));
+        instance.release(safeString2, null);
         assertCacheState(cache, 0, 1, nodeSize(1));
     }
 
@@ -409,34 +408,36 @@ public class AccordStateCacheTest
     public void evictionBlockedOnSaving()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 3 + nodeSize(3), cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), nodeSize(1) * 3 + nodeSize(3), cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString item = instance.acquire(Integer.toString(0));
-        testLoad(executor, item, Integer.toString(0));
+        testLoad(executor, instance, item, Integer.toString(0));
         item.set("0*");
         Assert.assertTrue(instance.isReferenced(item.key()));
-        instance.release(item);
+        instance.release(item, null);
 
         for (int i=1; i<4; i++)
         {
             item = instance.acquire(Integer.toString(i));
-            testLoad(executor, item, Integer.toString(i));
+            testLoad(executor, instance, item, Integer.toString(i));
             Assert.assertTrue(instance.isReferenced(item.key()));
-            instance.release(item);
+            instance.release(item, null);
         }
 
         assertCacheState(cache, 0, 4, nodeSize(1) * 3 + nodeSize(2));
-        assertCacheMetrics(cache.metrics, 0, 4, 4);
-        assertCacheMetrics(instance.instanceMetrics, 0, 4, 4);
+        assertCacheMetrics(cache.metrics, 0, 4, 4, 5);
+        assertCacheMetrics(type.typeMetrics, 0, 4, 4, 5);
 
         // force cache eviction
+        instance.acquire(Integer.toString(0));
         cache.setCapacity(0);
 
         // all should have been evicted except 0
-        assertCacheState(cache, 0, 1, nodeSize(2));
+        assertCacheState(cache, 1, 1, nodeSize(2));
 
         Assert.assertTrue(instance.keyIsCached("0", SafeString.class));
         Assert.assertFalse(instance.keyIsCached("1", SafeString.class));
@@ -448,28 +449,29 @@ public class AccordStateCacheTest
     public void testUpdates()
     {
         ManualExecutor executor = new ManualExecutor();
-        AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
-        AccordStateCache.Instance<String, String, SafeString> instance =
-            cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (current) -> null, (k, v) -> true, String::length);
+        AccordCache cache = new AccordCache(wrap(executor), OnSaved.immediate(), 500, cacheMetrics);
+        AccordCache.Type<String, String, SafeString> type =
+        cache.newType(String.class, (s, k) -> k, (s, k, c, o) -> null, Function.identity(), (s, k, v) -> true, String::length, SafeString::new);
+        AccordCache.Type<String, String, SafeString>.Instance instance = type.newInstance(null);
         assertCacheState(cache, 0, 0, 0);
 
         SafeString safeString = instance.acquire("1");
-        testLoad(executor, safeString, "1");
-        assertCacheState(cache, 1, 1, emptyNodeSize());
+        testLoad(executor, instance, safeString, "1");
+        assertCacheState(cache, 1, 1, nodeSize(1));
         Assert.assertNull(cache.head());
         Assert.assertNull(cache.tail());
 
         Assert.assertTrue(instance.isReferenced(safeString.key()));
-        assertCacheState(cache, 1, 1, emptyNodeSize());
+        assertCacheState(cache, 1, 1, nodeSize(1));
 
         safeString.set("11");
-        instance.release(safeString);
+        instance.release(safeString, null);
         assertCacheState(cache, 0, 1, nodeSize(2));
         Assert.assertSame(safeString.global, cache.head());
         Assert.assertSame(safeString.global, cache.tail());
 
-        assertCacheMetrics(cache.metrics, 0, 1, 1);
-        assertCacheMetrics(instance.instanceMetrics, 0, 1, 1);
+        assertCacheMetrics(cache.metrics, 0, 1, 1, 2);
+        assertCacheMetrics(type.typeMetrics, 0, 1, 1, 2);
     }
 
     private CacheSize mockCacheSize(long capacity, long size, int entries)
@@ -495,4 +497,9 @@ public class AccordStateCacheTest
         assertThat(integerInstance1).isSameAs(integerInstance2);
         assertThat(stringInstance1).isNotSameAs(integerInstance1);
     }
+    
+    private static Function<Runnable, Cancellable> wrap(ExecutorPlus executor)
+    {
+        return r -> AccordExecutor.wrap(executor.submit(r));
+    }
 }
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCachingStateTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCachingStateTest.java
deleted file mode 100644
index cf5ab6e8af..0000000000
--- a/test/unit/org/apache/cassandra/service/accord/AccordCachingStateTest.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.util.function.BiConsumer;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.cassandra.concurrent.ManualExecutor;
-import org.apache.cassandra.service.accord.AccordCachingState.Status;
-
-public class AccordCachingStateTest
-{
-    static class CachingState extends AccordCachingState<String, String>
-    {
-        public CachingState(String key, int index)
-        {
-            super(key, index);
-        }
-
-        public CachingState(String key)
-        {
-            this(key, 0);
-        }
-    }
-
-    static class InspectableCallback<V> implements BiConsumer<V, Throwable>
-    {
-        boolean called;
-        V result;
-        Throwable failure;
-
-        @Override
-        public void accept(V result, Throwable failure)
-        {
-            Assert.assertFalse(called);
-            called = true;
-            this.result = result;
-            this.failure = failure;
-        }
-    }
-
-    private static void assertIllegalState(Runnable runnable)
-    {
-        try
-        {
-            runnable.run();
-            Assert.fail("Expected IllegalStateException");
-        }
-        catch (IllegalStateException ise)
-        {
-            // expected
-        }
-    }
-
-    @Test
-    public void loadSuccessTest()
-    {
-        ManualExecutor executor = new ManualExecutor();
-        CachingState state = new CachingState("K");
-
-        Assert.assertEquals(Status.UNINITIALIZED, state.status());
-        assertIllegalState(state::get);
-        assertIllegalState(() -> state.set("VVVV"));
-        assertIllegalState(state::loading);
-
-        state.load(executor, k -> {
-            Assert.assertEquals("K", k);
-            return "V";
-        });
-        Assert.assertEquals(Status.LOADING, state.status());
-
-        executor.runOne();
-        Assert.assertEquals(Status.LOADED, state.status());
-        Assert.assertEquals("V", state.get());
-
-        assertIllegalState(() -> state.load(executor, k -> "CCC"));
-        assertIllegalState(state::loading);
-    }
-
-    @Test
-    public void loadNullTest()
-    {
-        ManualExecutor executor = new ManualExecutor();
-        CachingState state = new CachingState("K");
-        Assert.assertEquals(Status.UNINITIALIZED, state.status());
-
-        assertIllegalState(state::get);
-        assertIllegalState(() -> state.set("VVVV"));
-        assertIllegalState(state::loading);
-
-        state.load(executor, k -> {
-            Assert.assertEquals("K", k);
-            return null;
-        });
-        Assert.assertEquals(Status.LOADING, state.status());
-
-        executor.runOne();
-        Assert.assertEquals(Status.LOADED, state.status());
-        Assert.assertNull(state.get());
-
-        assertIllegalState(() -> state.load(executor, k -> "CCC"));
-        assertIllegalState(state::loading);
-    }
-
-    @Test
-    public void additionalCallbackTest()
-    {
-        ManualExecutor executor = new ManualExecutor();
-        CachingState state = new CachingState("K");
-        Assert.assertEquals(Status.UNINITIALIZED, state.status());
-
-        assertIllegalState(state::get);
-        assertIllegalState(() -> state.set("VVVV"));
-        assertIllegalState(state::loading);
-
-        state.load(executor, k -> {
-            Assert.assertEquals("K", k);
-            return "V";
-        });
-        Assert.assertEquals(Status.LOADING, state.status());
-
-        // register other callbacks
-        InspectableCallback<Object> callback1 = new InspectableCallback<>();
-        InspectableCallback<Object> callback2 = new InspectableCallback<>();
-
-        Assert.assertEquals(Status.LOADING, state.status());
-        state.loading().addCallback(callback1);
-        executor.runOne();
-        state.loading().addCallback(callback2);
-
-        Assert.assertTrue(callback1.called);
-        Assert.assertNull(callback1.failure);
-
-        Assert.assertTrue(callback2.called);
-        Assert.assertNull(callback2.failure);
-
-        Assert.assertEquals(Status.LOADED, state.status());
-        Assert.assertEquals("V", state.get());
-
-        assertIllegalState(() -> state.load(executor, k -> "CCC"));
-        assertIllegalState(state::loading);
-    }
-
-    @Test
-    public void loadFailureTest()
-    {
-        ManualExecutor executor = new ManualExecutor();
-        CachingState state = new CachingState("K");
-
-        Assert.assertEquals(Status.UNINITIALIZED, state.status());
-        assertIllegalState(state::get);
-        assertIllegalState(() -> state.set("VVVV"));
-        assertIllegalState(state::loading);
-
-        state.load(executor, k -> {
-            throw new RuntimeException();
-        });
-        Assert.assertEquals(Status.LOADING, state.status());
-
-        executor.runOne();
-        Assert.assertEquals(Status.FAILED_TO_LOAD, state.status());
-        assertIllegalState(state::get);
-        Assert.assertTrue(state.failure() instanceof RuntimeException);
-
-        assertIllegalState(() -> state.load(executor, k -> "CCC"));
-        assertIllegalState(state::loading);
-    }
-}
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
index e41e4ca579..d6f415789d 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
@@ -188,9 +188,9 @@ public class AccordCommandStoreTest
         cfk.set(cfk.current().update(command1).cfk());
         cfk.set(cfk.current().update(command2).cfk());
 
-        AccordKeyspace.getTimestampsForKeyMutation(commandStore, tfk, commandStore.nextSystemTimestampMicros()).apply();
+        AccordKeyspace.getTimestampsForKeyUpdater(commandStore, tfk.current(), commandStore.nextSystemTimestampMicros()).run();
         logger.info("E: {}", tfk);
-        TimestampsForKey actual = AccordKeyspace.loadTimestampsForKey(commandStore, key);
+        TimestampsForKey actual = AccordKeyspace.loadTimestampsForKey(commandStore.id(), key);
         logger.info("A: {}", actual);
 
         Assert.assertEquals(tfk.current(), actual);
@@ -219,9 +219,9 @@ public class AccordCommandStoreTest
         cfk.set(cfk.current().update(command1).cfk());
         cfk.set(cfk.current().update(command2).cfk());
 
-        AccordKeyspace.getCommandsForKeyMutation(commandStore.id(), cfk.current(), commandStore.nextSystemTimestampMicros()).apply();
+        AccordKeyspace.getCommandsForKeyUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run();
         logger.info("E: {}", cfk);
-        CommandsForKey actual = AccordKeyspace.loadCommandsForKey(commandStore, key);
+        CommandsForKey actual = AccordKeyspace.loadCommandsForKey(commandStore.id(), key);
         logger.info("A: {}", actual);
 
         Assert.assertEquals(cfk.current(), actual);
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java
index 4a53512862..72c5f07bc4 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java
@@ -93,7 +93,7 @@ public class AccordCommandTest
     public void basicCycleTest() throws Throwable
     {
         AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-        getUninterruptibly(commandStore.execute(PreLoadContext.empty(), unused -> commandStore.cache().setCapacity(0)));
+        getUninterruptibly(commandStore.execute(PreLoadContext.empty(), unused -> commandStore.executor().cacheUnsafe().setCapacity(0)));
 
         TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
         Txn txn = createWriteTxn(1);
@@ -168,7 +168,7 @@ public class AccordCommandTest
         Commit commit = Commit.SerializerSupport.create(txnId, route, 1, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute, null);
         getUninterruptibly(commandStore.execute(commit, commit::apply));
 
-        getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key).toParticipants(), KeyHistory.COMMANDS), safeStore -> {
+        getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key).toParticipants(), KeyHistory.SYNC), safeStore -> {
             Command before = safeStore.ifInitialised(txnId).current();
             Assert.assertEquals(commit.executeAt, before.executeAt());
             Assert.assertTrue(before.hasBeen(Status.Committed));
@@ -185,7 +185,7 @@ public class AccordCommandTest
     public void computeDeps() throws Throwable
     {
         AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-        getUninterruptibly(commandStore.execute(PreLoadContext.empty(), unused -> commandStore.cache().setCapacity(0)));
+        getUninterruptibly(commandStore.execute(PreLoadContext.empty(), unused -> commandStore.executor().cacheUnsafe().setCapacity(0)));
 
         TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1);
         Txn txn = createWriteTxn(2);
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java b/test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java
index 33b8419a6d..831e078eae 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java
@@ -26,6 +26,9 @@ import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import accord.local.Command;
+import accord.local.CommonAttributes;
+import accord.primitives.Ballot;
 import accord.primitives.TxnId;
 import accord.utils.AccordGens;
 import accord.utils.RandomSource;
@@ -39,10 +42,12 @@ import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.utils.StorageCompatibilityMode;
 
 import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
+import static org.apache.cassandra.service.accord.SavedCommand.getFlags;
 
 public class AccordJournalOrderTest
 {
@@ -63,7 +68,7 @@ public class AccordJournalOrderTest
     {
         if (new File(DatabaseDescriptor.getAccordJournalDirectory()).exists())
             ServerTestUtils.cleanupDirectory(DatabaseDescriptor.getAccordJournalDirectory());
-        AccordJournal accordJournal = new AccordJournal(TestParams.INSTANCE);
+        AccordJournal accordJournal = new AccordJournal(TestParams.INSTANCE, new AccordAgent());
         accordJournal.start(null);
         RandomSource randomSource = RandomSource.wrap(new Random());
         TxnId id1 = AccordGens.txnIds().next(randomSource);
@@ -75,15 +80,16 @@ public class AccordJournalOrderTest
             TxnId txnId = randomSource.nextBoolean() ? id1 : id2;
             JournalKey key = new JournalKey(txnId, JournalKey.Type.COMMAND_DIFF, randomSource.nextInt(5));
             res.compute(key, (k, prev) -> prev == null ? 1 : prev + 1);
+            Command command = Command.NotDefined.notDefined(new CommonAttributes.Mutable(txnId), Ballot.ZERO);
             accordJournal.appendCommand(key.commandStoreId,
-                                        new SavedCommand.DiffWriter(txnId, null, null),
+                                        new SavedCommand.Writer(command, getFlags(null, command)),
                                         () -> {});
         }
 
         Runnable check = () -> {
             for (JournalKey key : res.keySet())
             {
-                SavedCommand.Builder diffs = accordJournal.loadDiffs(key.commandStoreId, (TxnId) key.id);
+                SavedCommand.Builder diffs = accordJournal.loadDiffs(key.commandStoreId, key.id);
                 Assert.assertEquals(String.format("%d != %d for key %s", diffs.count(), res.get(key).intValue(), key),
                                     diffs.count(), res.get(key).intValue());
             }
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java
index 82e8f25afb..f5b133a7ba 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java
@@ -50,9 +50,6 @@ import accord.primitives.Status;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.Observable;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
@@ -148,7 +145,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
                                                      .sorted()
                                                      .collect(Collectors.toList());
 
-        qt().check(rs -> {
+        qt().withSeed(3447657054093411240L).check(rs -> {
             AccordKeyspace.unsafeClear();
             // control SSTable format
             setSelectedSSTableFormat(sstableFormats.get(rs.pick(sstableFormatNames)));
@@ -285,8 +282,8 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
                     TokenKey start = expected.get(0);
                     TokenKey end = expected.get(expected.size() - 1);
 
-                    AsyncChain<List<TokenKey>> map = Observable.asChain(callback -> AccordKeyspace.findAllKeysBetween(store, start, true, end, true, callback));
-                    List<TokenKey> actual = AsyncChains.getUnchecked(map);
+                    List<TokenKey> actual = new ArrayList<>();
+                    AccordKeyspace.findAllKeysBetween(store, start, true, end, true, actual::add);
                     Assertions.assertThat(actual).isEqualTo(expected);
                 }
 
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java
index 491f193fac..6a9a055822 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java
@@ -63,7 +63,7 @@ public class AccordMessageSinkTest
     private static final Topologies topologies = new Topologies.Single((TopologySorter) (StaticSorter)(a, b, ignore) -> 0, topology);
 
     private static final MessageDelivery messaging = Mockito.mock(MessageDelivery.class);
-    private static final AccordMessageSink sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging, mapping, new RequestCallbacks(new AccordTimeService()));
+    private static AccordMessageSink sink;
 
     @BeforeClass
     public static void setup()
@@ -71,6 +71,7 @@ public class AccordMessageSinkTest
         DatabaseDescriptor.clientInitialization();
         DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
         ClusterMetadataService.initializeForClients();
+        sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging, mapping, new RequestCallbacks(new AccordTimeService()));
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java
similarity index 70%
rename from test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java
rename to test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java
index 20f123572f..bc476baab5 100644
--- a/test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java
@@ -16,25 +16,27 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord.async;
+package org.apache.cassandra.service.accord;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.LockSupport;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 import accord.local.StoreParticipants;
 import accord.primitives.Participants;
 import accord.primitives.Route;
-import accord.utils.DefaultRandom;
+
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
-import org.apache.cassandra.concurrent.SimulatedExecutorFactory;
-import org.apache.cassandra.concurrent.Stage;
 
 import org.junit.Assert;
 import org.junit.Before;
@@ -73,13 +75,8 @@ import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordCachingState;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeCommandStore;
-import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.AccordTestUtils;
+import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
+import org.apache.cassandra.service.accord.AccordExecutor.ExclusiveGlobalCaches;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.utils.AssertionUtils;
@@ -89,7 +86,7 @@ import org.assertj.core.api.Assertions;
 import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 
-import static accord.local.KeyHistory.COMMANDS;
+import static accord.local.KeyHistory.SYNC;
 import static accord.local.PreLoadContext.contextFor;
 import static accord.utils.Property.qt;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
@@ -99,11 +96,10 @@ import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialT
 import static org.apache.cassandra.service.accord.AccordTestUtils.keys;
 import static org.apache.cassandra.service.accord.AccordTestUtils.loaded;
 import static org.apache.cassandra.service.accord.AccordTestUtils.txnId;
-import static org.apache.cassandra.service.accord.async.AsyncLoader.txnIds;
 
-public class AsyncOperationTest
+public class AccordTaskTest
 {
-    private static final Logger logger = LoggerFactory.getLogger(AsyncOperationTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(AccordTaskTest.class);
     private static final AtomicLong clock = new AtomicLong(0);
 
     @BeforeClass
@@ -132,14 +128,12 @@ public class AsyncOperationTest
     {
         AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
         TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        Txn txn = AccordTestUtils.createWriteTxn((int)clock.incrementAndGet());
-        PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
 
         getUninterruptibly(commandStore.execute(contextFor(txnId), instance -> {
             // TODO review: This change to `ifInitialized` was done in a lot of places and it doesn't preserve this property
             // I fixed this reference to point to `ifLoadedAndInitialised` and but didn't update other places
-            SafeCommand command = instance.ifLoadedAndInitialised(txnId);
-            Assert.assertNull(command);
+            Assert.assertNull(instance.ifInitialised(txnId));
+            Assert.assertNull(instance.ifLoadedAndInitialisedAndNotErased(txnId));
         }));
 
         UntypedResultSet result = AccordKeyspace.loadCommandRow(commandStore, txnId);
@@ -170,7 +164,7 @@ public class AsyncOperationTest
         TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
 
         getUninterruptibly(commandStore.execute(contextFor(key), instance -> {
-            SafeCommandsForKey cfk = ((AccordSafeCommandStore) instance).maybeCommandsForKey(key);
+            SafeCommandsForKey cfk = instance.ifLoadedAndInitialised(key);
             Assert.assertNull(cfk);
         }));
 
@@ -215,7 +209,7 @@ public class AsyncOperationTest
 
         try
         {
-            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, route, COMMANDS), safe -> {
+            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, route, SYNC), safe -> {
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore));
                 CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore));
                 return safe.ifInitialised(txnId).current();
@@ -223,12 +217,17 @@ public class AsyncOperationTest
 
             // clear cache
             commandStore.executeBlocking(() -> {
-                long cacheSize = commandStore.cache().capacity();
-                commandStore.cache().setCapacity(0);
-                commandStore.cache().setCapacity(cacheSize);
-                commandStore.cache().awaitSaveResults();
+                try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();)
+                {
+                    long cacheSize = cache.global.capacity();
+                    cache.global.setCapacity(0);
+                    cache.global.setCapacity(cacheSize);
+                }
             });
 
+            while (commandStore.executor().hasTasks())
+                LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100));
+
             return command;
         }
         catch (ExecutionException e)
@@ -262,7 +261,7 @@ public class AsyncOperationTest
 
         try
         {
-            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, route, COMMANDS), safe -> {
+            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, route, SYNC), safe -> {
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, appendDiffToLog(commandStore));
                 CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, executeAt, deps, appendDiffToLog(commandStore));
                 CheckedCommands.commit(safe, SaveStatus.Committed, Ballot.ZERO, txnId, route, partialTxn, executeAt, deps, appendDiffToLog(commandStore));
@@ -272,12 +271,17 @@ public class AsyncOperationTest
 
             // clear cache
             commandStore.executeBlocking(() -> {
-                long cacheSize = commandStore.cache().capacity();
-                commandStore.cache().setCapacity(0);
-                commandStore.cache().setCapacity(cacheSize);
-                commandStore.cache().awaitSaveResults();
+                try (ExclusiveGlobalCaches cache = commandStore.executor().lockCaches();)
+                {
+                    long cacheSize = cache.global.capacity();
+                    cache.global.setCapacity(0);
+                    cache.global.setCapacity(cacheSize);
+                }
             });
 
+            while (commandStore.executor().hasTasks())
+                LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100));
+
             return command;
         }
         catch (ExecutionException e)
@@ -286,78 +290,6 @@ public class AsyncOperationTest
         }
     }
 
-    private static void assertFutureState(AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> cache, TxnId txnId, boolean referenceExpected, boolean expectLoadFuture, boolean expectSaveFuture)
-    {
-        if (cache.isReferenced(txnId) != referenceExpected)
-            throw new AssertionError(referenceExpected ? "Cache reference unexpectedly not found for " + txnId
-                                                       : "Unexpectedly found cache reference for " + txnId);
-        cache.complete(txnId);
-        if (cache.hasLoadResult(txnId) != expectLoadFuture)
-            throw new AssertionError(expectLoadFuture ? "Load future unexpectedly not found for " + txnId
-                                                      : "Unexpectedly found load future for " + txnId);
-        if (cache.hasSaveResult(txnId) != expectSaveFuture)
-            throw new AssertionError(expectSaveFuture ? "Save future unexpectedly not found for " + txnId
-                                                      : "Unexpectedly found save future for " + txnId);
-
-    }
-
-    /**
-     * save and load futures should be cleaned up as part of the operation
-     */
-    @Test
-    public void testFutureCleanup() throws Throwable
-    {
-        SimulatedExecutorFactory factory = new SimulatedExecutorFactory(new DefaultRandom(42), 42);
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", factory.scheduled("ignored"), Stage.MUTATION.executor());
-
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-
-        createStableAndPersist(commandStore, txnId);
-
-        Consumer<SafeCommandStore> consumer = safeStore -> safeStore.ifInitialised(txnId).readyToExecute(safeStore);
-        PreLoadContext ctx = contextFor(txnId);
-        AsyncOperation<Void> operation = new AsyncOperation.ForConsumer(commandStore, ctx, consumer)
-        {
-
-            private AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> cache()
-            {
-                return commandStore.commandCache();
-            }
-
-            @Override
-            AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-            {
-                return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys(), preLoadContext.keyHistory())
-                {
-                    @Override
-                    void state(State state)
-                    {
-                        switch (state)
-                        {
-                            case SETUP:
-                                assertFutureState(cache(), txnId, false, false, false);
-                                factory.processAll();
-                                break;
-                            case FINISHED:
-                                assertFutureState(cache(), txnId, true, false, false);
-                                factory.processAll();
-                                break;
-                            case LOADING:
-                                assertFutureState(cache(), txnId, true, true, false);
-                                factory.processAll();
-                                break;
-                        }
-                        super.state(state);
-                    }
-                };
-            }
-        };
-
-        commandStore.executor().submit(operation);
-
-        getUninterruptibly(operation);
-    }
-
     @Test
     public void loadFail()
     {
@@ -365,12 +297,12 @@ public class AsyncOperationTest
         // all txn use the same key; 0
         Keys keys = keys(Schema.instance.getTableMetadata("ks", "tbl"), 0);
         AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-        commandStore.executeBlocking(() -> commandStore.cache().setCapacity(0));
+        commandStore.executeBlocking(() -> commandStore.executor().cacheUnsafe().setCapacity(0));
         Gen<TxnId> txnIdGen = rs -> txnId(1, clock.incrementAndGet(), 1);
 
-        qt().withPure(false)
+        qt().withSeed(3447647345436261108L).withPure(false)
             .withExamples(50)
-            .forAll(Gens.random(), Gens.lists(txnIdGen).ofSizeBetween(1, 10))
+            .forAll(Gens.random(), Gens.lists(txnIdGen).ofSizeBetween(1, 2))
             .check((rs, ids) -> {
             before(); // truncate tables
 
@@ -380,23 +312,25 @@ public class AsyncOperationTest
             awaitDone(commandStore, ids, participants);
             assertNoReferences(commandStore, ids, participants);
 
-            PreLoadContext ctx = contextFor(null, ids, participants, COMMANDS);
+            PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC);
             Consumer<SafeCommandStore> consumer = Mockito.mock(Consumer.class);
 
             Map<TxnId, Boolean> failed = selectFailedTxn(rs, ids);
-            commandStore.commandCache().unsafeSetLoadFunction(txnId ->
+            try (ExclusiveGlobalCaches caches = commandStore.executor().lockCaches())
             {
-                logger.info("Attempting to load {}; expected to fail? {}", txnId, failed.get(txnId));
-                if (!failed.get(txnId))
-                    return commandStore.loadCommand(txnId);
-                throw new NullPointerException("txn_id " + txnId);
-            });
-            AsyncOperation<Void> o1 = new AsyncOperation.ForConsumer(commandStore, ctx, consumer);
-
-            AssertionUtils.assertThatThrownBy(() -> getUninterruptibly(o1))
-                      .hasRootCause()
-                      .isInstanceOf(NullPointerException.class)
-                      .hasNoSuppressedExceptions();
+                caches.commands.unsafeSetLoadFunction((s, txnId) ->
+                {
+                    logger.info("Attempting to load {}; expected to fail? {}", txnId, failed.get(txnId));
+                    if (!failed.get(txnId))
+                        return commandStore.loadCommand(txnId);
+                    throw new NullPointerException("txn_id " + txnId);
+                });
+            }
+            AccordTask<Void> o1 = AccordTask.create(commandStore, ctx, consumer);
+            AssertionUtils.assertThatThrownBy(() -> getUninterruptibly(o1.chain()))
+                          .hasRootCause()
+                          .isInstanceOf(NullPointerException.class)
+                          .hasNoSuppressedExceptions();
 
             Mockito.verifyNoInteractions(consumer);
 
@@ -406,16 +340,19 @@ public class AsyncOperationTest
             awaitDone(commandStore, ids, participants);
 
             // can we recover?
-            commandStore.commandCache().unsafeSetLoadFunction(txnId -> {
-                Command cmd = commandStore.loadCommand(txnId);
-                return cmd;
-            });
-            AsyncOperation.ForConsumer o2 = new AsyncOperation.ForConsumer(commandStore, ctx, store -> {
+            try (ExclusiveGlobalCaches caches = commandStore.executor().lockCaches())
+            {
+                caches.commands.unsafeSetLoadFunction((s, txnId) -> {
+                    Command cmd = commandStore.loadCommand(txnId);
+                    return cmd;
+                });
+            }
+            AccordTask<Void> o2 = AccordTask.create(commandStore, ctx, store -> {
                 ids.forEach(id -> {
                     store.ifInitialised(id).readyToExecute(store);
                 });
             });
-            getUninterruptibly(o2);
+            getUninterruptibly(o2.chain());
             awaitDone(commandStore, ids, participants);
             assertNoReferences(commandStore, ids, participants);
 
@@ -440,15 +377,15 @@ public class AsyncOperationTest
             assertNoReferences(commandStore, ids, participants);
             createCommand(commandStore, rs, ids);
 
-            PreLoadContext ctx = contextFor(null, ids, participants, COMMANDS);
+            PreLoadContext ctx = contextFor(ids.get(0), ids.size() == 1 ? null : ids.get(1), participants, SYNC);
 
             Consumer<SafeCommandStore> consumer = Mockito.mock(Consumer.class);
             String errorMsg = "txn_ids " + ids;
             Mockito.doThrow(new NullPointerException(errorMsg)).when(consumer).accept(Mockito.any());
 
-            AsyncOperation<Void> operation = new AsyncOperation.ForConsumer(commandStore, ctx, consumer);
+            AccordTask<Void> operation = AccordTask.create(commandStore, ctx, consumer);
 
-            AssertionUtils.assertThatThrownBy(() -> getUninterruptibly(operation))
+            AssertionUtils.assertThatThrownBy(() -> getUninterruptibly(operation.chain()))
                           .hasRootCause()
                           .isInstanceOf(NullPointerException.class)
                           .hasMessage(errorMsg)
@@ -479,28 +416,31 @@ public class AsyncOperationTest
 
     private static Map<TxnId, Boolean> selectFailedTxn(RandomSource rs, List<TxnId> ids)
     {
+        ids = new ArrayList<>(ids);
         Map<TxnId, Boolean> failed = Maps.newHashMapWithExpectedSize(ids.size());
-        for (TxnId id : ids)
-            failed.put(id, rs.nextBoolean());
-        if (failed.values().stream().allMatch(b -> b == Boolean.FALSE))
-            failed.put(ids.get(0), Boolean.TRUE);
+        int failedCount = Math.max(1, rs.nextInt(ids.size()));
+        Collections.shuffle(ids, rs.asJdkRandom());
+        for (int i = 0 ; i < failedCount ; ++i)
+            failed.put(ids.get(i), true);
+        for (int i = failedCount ; i < ids.size() ; ++i)
+            failed.put(ids.get(i), false);
         return failed;
     }
 
     private static void assertNoReferences(AccordCommandStore commandStore, List<TxnId> ids, Participants<RoutingKey> keys)
     {
         AssertionError error = null;
-        try
+        try (ExclusiveCaches caches = commandStore.lockCaches())
         {
-            assertNoReferences(commandStore.commandCache(), ids);
+            assertNoReferences(caches.commands(), ids);
         }
         catch (AssertionError e)
         {
             error = e;
         }
-        try
+        try (ExclusiveCaches caches = commandStore.lockCaches())
         {
-            assertNoReferences(commandStore.commandsForKeyCache(), keys);
+            assertNoReferences(caches.commandsForKeys(), keys);
         }
         catch (AssertionError e)
         {
@@ -510,16 +450,18 @@ public class AsyncOperationTest
         if (error != null) throw error;
     }
 
-    private static <T> void assertNoReferences(AccordStateCache.Instance<T, ?, ?> cache, Iterable<T> keys)
+    private static <T> void assertNoReferences(AccordCache.Type<T, ?, ?>.Instance cache, Iterable<T> keys)
     {
         AssertionError error = null;
         for (T key : keys)
         {
-            AccordCachingState<T, ?> node = cache.getUnsafe(key);
+            AccordCacheEntry<T, ?> node = cache.getUnsafe(key);
             if (node == null) continue;
             try
             {
-                Assertions.assertThat(node.referenceCount())
+                if (node.references() > 0)
+                    throw new IllegalStateException();
+                Assertions.assertThat(node.references())
                           .describedAs("Key %s found referenced in cache", key)
                           .isEqualTo(0);
             }
@@ -540,15 +482,15 @@ public class AsyncOperationTest
 
     private static void awaitDone(AccordCommandStore commandStore, List<TxnId> ids, Participants<RoutingKey> keys)
     {
-        awaitDone(commandStore.commandCache(), ids);
-        awaitDone(commandStore.commandsForKeyCache(), keys);
+        awaitDone(commandStore.cachesUnsafe().commands(), ids);
+        awaitDone(commandStore.cachesUnsafe().commandsForKeys(), keys);
     }
 
-    private static <T> void awaitDone(AccordStateCache.Instance<T, ?, ?> cache, Iterable<T> keys)
+    private static <T> void awaitDone(AccordCache.Type<T, ?, ?>.Instance cache, Iterable<T> keys)
     {
         for (T key : keys)
         {
-            AccordCachingState<T, ?> node = cache.getUnsafe(key);
+            AccordCacheEntry<T, ?> node = cache.getUnsafe(key);
             if (node == null) continue;
             Awaitility.await("For node " + node.key() + " to complete")
             .atMost(Duration.ofMinutes(1))
diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
index a40200a6a6..dbd42297cc 100644
--- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
+++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
@@ -35,12 +35,13 @@ import com.google.common.collect.Sets;
 import org.junit.Assert;
 
 import accord.api.Data;
-import accord.api.LocalListeners;
 import accord.api.ProgressLog.NoOpProgressLog;
-import accord.api.RemoteListeners;
+import accord.api.RemoteListeners.NoOpRemoteListeners;
 import accord.api.Result;
 import accord.api.RoutingKey;
+import accord.api.Timeouts;
 import accord.impl.DefaultLocalListeners;
+import accord.impl.DefaultLocalListeners.NotifySink.NoOpNotifySink;
 import accord.impl.InMemoryCommandStore;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -52,7 +53,6 @@ import accord.local.Node.Id;
 import accord.local.NodeCommandStoreService;
 import accord.local.TimeService;
 import accord.local.PreLoadContext;
-import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.local.StoreParticipants;
 import accord.primitives.Ballot;
@@ -77,11 +77,11 @@ import accord.utils.SortedArrays.SortedArrayList;
 import accord.utils.async.AsyncChains;
 import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.concurrent.ManualExecutor;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.AccordSpec;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.DurationSpec;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.statements.TransactionStatement;
@@ -91,7 +91,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
@@ -108,7 +108,8 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import static accord.primitives.Routable.Domain.Key;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
 import static java.lang.String.format;
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
+import static org.apache.cassandra.service.accord.AccordExecutor.wrap;
 
 public class AccordTestUtils
 {
@@ -167,25 +168,16 @@ public class AccordTestUtils
         }
     }
 
-    public static <K, V> AccordCachingState<K, V> loaded(K key, V value, int index)
+    public static <K, V> AccordCacheEntry<K, V> loaded(K key, V value)
     {
-        AccordCachingState<K, V> global = new AccordCachingState<>(key, index);
-        global.load(ImmediateExecutor.INSTANCE, k -> {
-            Assert.assertEquals(key, k);
-            return value;
-        });
-        Assert.assertEquals(AccordCachingState.Status.LOADED, global.status());
+        AccordCacheEntry<K, V> global = new AccordCacheEntry<>(key, null);
+        global.initialize(value);
         return global;
     }
 
-    public static <K, V> AccordCachingState<K, V> loaded(K key, V value)
-    {
-        return loaded(key, value, 0);
-    }
-
     public static AccordSafeCommand safeCommand(Command command)
     {
-        AccordCachingState<TxnId, Command> global = loaded(command.txnId(), command);
+        AccordCacheEntry<TxnId, Command> global = loaded(command.txnId(), command);
         return new AccordSafeCommand(global);
     }
 
@@ -197,11 +189,13 @@ public class AccordTestUtils
         };
     }
 
-    public static <K, V> void testLoad(ManualExecutor executor, AccordSafeState<K, V> safeState, V val)
+    public static <K, V> void testLoad(ManualExecutor executor, AccordCache.Type<K, V, ?>.Instance instance, AccordSafeState<K, V> safeState, V val)
     {
-        Assert.assertEquals(AccordCachingState.Status.LOADING, safeState.globalStatus());
+        Assert.assertEquals(AccordCacheEntry.Status.WAITING_TO_LOAD, safeState.global().status());
+        safeState.global().load(wrap(executor), null, instance.parent().adapter(), AccordCacheEntry.OnLoaded.immediate());
+        Assert.assertEquals(AccordCacheEntry.Status.LOADING, safeState.global().status());
         executor.runOne();
-        Assert.assertEquals(AccordCachingState.Status.LOADED, safeState.globalStatus());
+        Assert.assertEquals(AccordCacheEntry.Status.LOADED, safeState.global().status());
         safeState.preExecute();
         Assert.assertEquals(val, safeState.current());
     }
@@ -371,8 +365,8 @@ public class AccordTestUtils
             private ToLongFunction<TimeUnit> elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now);
 
             @Override public Id id() { return node;}
+            @Override public Timeouts timeouts() { return null; }
             @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; }
-
             @Override public long epoch() {return 1; }
             @Override public long now() {return now.getAsLong(); }
             @Override public Timestamp uniqueNow() { return uniqueNow(Timestamp.NONE); }
@@ -389,11 +383,20 @@ public class AccordTestUtils
 
     public static AccordCommandStore createAccordCommandStore(
         Node.Id node, LongSupplier now, Topology topology, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor)
+    {
+        AccordAgent agent = new AccordAgent();
+        AccordExecutor executor = new AccordExecutorSyncSubmit(0, RUN_WITH_LOCK, CommandStore.class.getSimpleName() + '[' + 0 + ']', new AccordCacheMetrics("test"), loadExecutor, saveExecutor, loadExecutor, agent);
+        return createAccordCommandStore(node, now, topology, agent, executor);
+    }
+
+    public static AccordCommandStore createAccordCommandStore(
+        Node.Id node, LongSupplier now, Topology topology, AccordAgent agent, AccordExecutor executor)
     {
         NodeCommandStoreService time = new NodeCommandStoreService()
         {
             private ToLongFunction<TimeUnit> elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MICROSECONDS, this::now);
 
+            @Override public Timeouts timeouts() { return null; }
             @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; }
             @Override public Id id() { return node;}
             @Override public long epoch() {return 1; }
@@ -407,30 +410,18 @@ public class AccordTestUtils
 
         if (new File(DatabaseDescriptor.getAccordJournalDirectory()).exists())
             ServerTestUtils.cleanupDirectory(DatabaseDescriptor.getAccordJournalDirectory());
-        AccordJournal journal = new AccordJournal(new AccordSpec.JournalSpec());
+        AccordSpec.JournalSpec spec = new AccordSpec.JournalSpec();
+        spec.flushPeriod = new DurationSpec.IntSecondsBound(1);
+        AccordJournal journal = new AccordJournal(spec, agent);
         journal.start(null);
 
-        AccordStateCache stateCache = new AccordStateCache(loadExecutor, saveExecutor, 8 << 20, new AccordStateCacheMetrics("test"));
         SingleEpochRanges holder = new SingleEpochRanges(topology.rangesForNode(node));
-        AccordCommandStore result = new AccordCommandStore(0,
-                                                           time,
-                                                           new AccordAgent(),
-                                                           null,
+        AccordCommandStore result = new AccordCommandStore(0, time, agent, null,
                                                            cs -> new NoOpProgressLog(),
-                                                           cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
-                                                           {
-                                                               @Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
-                                                               @Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
-                                                           }),
-                                                           holder,
-                                                           journal,
-                                                           new AccordCommandStore.CommandStoreExecutor(stateCache, executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + 0 + ']')));
+                                                           cs -> new DefaultLocalListeners(new NoOpRemoteListeners(), new NoOpNotifySink()),
+                                                           holder, journal, executor);
         holder.set(result);
-
-        // TODO: CompactionAccordIteratorsTest relies on this
-        result.execute(PreLoadContext.empty(),
-                       result::updateRangesForEpoch)
-              .beginAsResult();
+        result.unsafeUpdateRangesForEpoch();
         return result;
     }
 
@@ -447,7 +438,7 @@ public class AccordTestUtils
         Node.Id node = new Id(1);
         Topology topology = new Topology(1, new Shard(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node), Collections.emptySet()));
         AccordCommandStore store = createAccordCommandStore(node, now, topology, loadExecutor, saveExecutor);
-        store.execute(PreLoadContext.empty(), safeStore -> ((AccordCommandStore)safeStore.commandStore()).cache().setCapacity(1 << 20));
+        store.execute(PreLoadContext.empty(), safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20));
         return store;
     }
 
@@ -520,7 +511,7 @@ public class AccordTestUtils
 
     public static void appendCommandsBlocking(AccordCommandStore commandStore, Command before, Command after)
     {
-        SavedCommand.DiffWriter diff = SavedCommand.diff(before, after);
+        SavedCommand.Writer diff = SavedCommand.diff(before, after);
         if (diff == null) return;
         Condition condition = Condition.newOneTimeCondition();
         commandStore.appendCommands(Collections.singletonList(diff), condition::signal);
diff --git a/test/unit/org/apache/cassandra/service/accord/CommandsForRangesTest.java b/test/unit/org/apache/cassandra/service/accord/CommandsForRangesTest.java
index 115a231d5f..b1862b7445 100644
--- a/test/unit/org/apache/cassandra/service/accord/CommandsForRangesTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/CommandsForRangesTest.java
@@ -30,10 +30,12 @@ import accord.impl.IntKey;
 import accord.primitives.SaveStatus;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.AccordGens;
 import accord.utils.Gen;
 import accord.utils.Gens;
+import org.apache.cassandra.service.accord.CommandsForRangesLoader.Summary;
 
 import static accord.utils.Property.qt;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -47,13 +49,13 @@ public class CommandsForRangesTest
     private static final Gen<CommandsForRanges> CFK_GEN = rs -> {
         Ranges ranges = RANGES_GEN.next(rs);
         int numTxn = 10;
-        TreeMap<TxnId, CommandsForRangesLoader.Summary> map = new TreeMap<>();
+        TreeMap<TxnId, Summary> map = new TreeMap<>();
         for (int i = 0; i < numTxn; i++)
         {
             TxnId id = TXN_ID_GEN.next(rs);
-            map.put(id, new CommandsForRangesLoader.Summary(id, id, SaveStatus.ReadyToExecute, ranges, null, false));
+            map.put(id, new Summary(id, id, SaveStatus.ReadyToExecute, ranges, null, false));
         }
-        return CommandsForRanges.create(ranges, map);
+        return CommandsForRanges.create(ranges, new TreeMap<Timestamp, Summary>(map));
     };
     private static final IntKey.Routing MIN = IntKey.routing(Integer.MIN_VALUE);
     private static final IntKey.Routing MAX = IntKey.routing(Integer.MAX_VALUE);
diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java
index b252d13f76..55e64e87a9 100644
--- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java
@@ -685,11 +685,11 @@ public class EpochSyncTest
                         EpochReady ready = new EpochReady(topology.epoch(), metadata, coordination, data, reads);
 
                         topology().onTopologyUpdate(topology, () -> ready);
-                        ready.coordination.addCallback(() -> topology().onEpochSyncComplete(id, topology.epoch()));
+                        ready.fastPath.addCallback(() -> topology().onEpochSyncComplete(id, topology.epoch()));
                         if (topology().minEpoch() == topology.epoch() && topology().epoch() != topology.epoch())
-                            return ready.coordination;
+                            return ready.fastPath;
                         config.acknowledgeEpoch(ready, startSync);
-                        return ready.coordination;
+                        return ready.fastPath;
                     }
 
                     @Override
diff --git a/test/unit/org/apache/cassandra/service/accord/MockJournal.java b/test/unit/org/apache/cassandra/service/accord/MockJournal.java
index 7d7c49153e..fa507673e4 100644
--- a/test/unit/org/apache/cassandra/service/accord/MockJournal.java
+++ b/test/unit/org/apache/cassandra/service/accord/MockJournal.java
@@ -136,13 +136,13 @@ public class MockJournal implements IJournal
     }
 
     @Override
-    public void appendCommand(int store, SavedCommand.DiffWriter diff, Runnable onFlush)
+    public void appendCommand(int store, SavedCommand.Writer diff, Runnable onFlush)
     {
         if (diff != null)
         {
             commands.computeIfAbsent(new JournalKey(diff.after().txnId(), JournalKey.Type.COMMAND_DIFF, store),
                                      (ignore_) -> new ArrayList<>())
-                    .add(diff(diff.before(), diff.after()));
+                    .add(diff(null, diff.after()));
         }
 
         if (onFlush != null)
diff --git a/test/unit/org/apache/cassandra/service/accord/SavedCommandTest.java b/test/unit/org/apache/cassandra/service/accord/SavedCommandTest.java
index 0e2c57bb97..1760286ba3 100644
--- a/test/unit/org/apache/cassandra/service/accord/SavedCommandTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/SavedCommandTest.java
@@ -97,7 +97,7 @@ public class SavedCommandTest
                     if (saveStatus == SaveStatus.TruncatedApplyWithDeps) continue;
                     out.clear();
                     Command orig = cmdBuilder.build(saveStatus);
-                    SavedCommand.serialize(null, orig, out, userVersion);
+                    SavedCommand.serialize(orig, getFlags(null, orig), out, userVersion);
                     SavedCommand.Builder builder = new SavedCommand.Builder(orig.txnId(), Load.ALL);
                     builder.deserializeNext(new DataInputBuffer(out.unsafeGetBufferAndFlip(), false), userVersion);
                     // We are not persisting the result, so force it for strict equality
diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
index f96c3b438e..88f282148f 100644
--- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
@@ -21,8 +21,10 @@ package org.apache.cassandra.service.accord;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
 import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -32,9 +34,12 @@ import accord.api.LocalListeners;
 import accord.api.ProgressLog;
 import accord.api.RemoteListeners;
 import accord.api.RoutingKey;
+import accord.api.Timeouts;
 import accord.impl.DefaultLocalListeners;
+import accord.impl.DefaultTimeouts;
 import accord.impl.SizeOfIntersectionSorter;
 import accord.impl.TestAgent;
+import accord.impl.basic.SimulatedFault;
 import accord.local.Command;
 import accord.local.CommandStore;
 import accord.local.CommandStores;
@@ -47,6 +52,7 @@ import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
 import accord.messages.BeginRecovery;
 import accord.messages.PreAccept;
+import accord.messages.Reply;
 import accord.messages.TxnRequest;
 import accord.primitives.AbstractUnseekableKeys;
 import accord.primitives.Ballot;
@@ -75,14 +81,13 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.memtable.Memtable;
-import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Generators;
 import org.apache.cassandra.utils.Pair;
 import org.assertj.core.api.Assertions;
 
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
 import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
 import static org.apache.cassandra.utils.AccordGenerators.fromQT;
@@ -95,7 +100,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
     private final BooleanSupplier shouldEvict, shouldFlush, shouldCompact;
 
     public final NodeCommandStoreService storeService;
-    public final AccordCommandStore store;
+    public final AccordCommandStore commandStore;
     public final Node.Id nodeId;
     public final Topology topology;
     public final Topologies topologies;
@@ -104,12 +109,26 @@ public class SimulatedAccordCommandStore implements AutoCloseable
     public final List<String> evictions = new ArrayList<>();
     public Predicate<Throwable> ignoreExceptions = ignore -> false;
 
+    public interface FunctionWrapper
+    {
+        <I1, I2, O> BiFunction<I1, I2, O> wrap(BiFunction<I1, I2, O> f);
+
+        static <I1, I2, O> BiFunction<I1, I2, O> identity(BiFunction<I1, I2, O> f) { return f; }
+        static FunctionWrapper identity() { return FunctionWrapper::identity; }
+    }
+
+
     public SimulatedAccordCommandStore(RandomSource rs)
+    {
+        this(rs, FunctionWrapper.identity());
+    }
+
+    public SimulatedAccordCommandStore(RandomSource rs, FunctionWrapper loadFunctionWrapper)
     {
         globalExecutor = new SimulatedExecutorFactory(rs.fork(), fromQT(Generators.TIMESTAMP_GEN.map(java.sql.Timestamp::getTime)).mapToLong(TimeUnit.MILLISECONDS::toNanos).next(rs), failures::add);
         this.unorderedScheduled = globalExecutor.scheduled("ignored");
         ExecutorFactory.Global.unsafeSet(globalExecutor);
-        for (Stage stage : Arrays.asList(Stage.READ, Stage.MUTATION, Stage.ACCORD_RANGE_LOADER))
+        for (Stage stage : Arrays.asList(Stage.READ, Stage.MUTATION))
             stage.unsafeSetExecutor(unorderedScheduled);
         for (Stage stage : Arrays.asList(Stage.MISC, Stage.ACCORD_MIGRATION, Stage.READ, Stage.MUTATION))
             stage.unsafeSetExecutor(globalExecutor.configureSequential("ignore").build());
@@ -119,6 +138,9 @@ public class SimulatedAccordCommandStore implements AutoCloseable
         this.storeService = new NodeCommandStoreService()
         {
             private final ToLongFunction<TimeUnit> elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.NANOSECONDS, this::now);
+            final Timeouts timeouts = new DefaultTimeouts(this);
+
+            @Override public Timeouts timeouts() { return timeouts; }
 
             @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; }
 
@@ -162,71 +184,70 @@ public class SimulatedAccordCommandStore implements AutoCloseable
             }
         };
 
-        AccordStateCache stateCache = new AccordStateCache(Stage.READ.executor(), Stage.MUTATION.executor(), 8 << 20, new AccordStateCacheMetrics("test"));
         this.journal = new MockJournal();
-        this.store = new AccordCommandStore(0,
-                                            storeService,
-                                            new TestAgent.RethrowAgent()
-                                            {
-                                                @Override
-                                                public long preAcceptTimeout()
-                                                {
-                                                    return Long.MAX_VALUE;
-                                                }
+        TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent()
+        {
+            @Override
+            public long preAcceptTimeout()
+            {
+                return Long.MAX_VALUE;
+            }
 
-                                                @Override
-                                                public void onUncaughtException(Throwable t)
-                                                {
-                                                    if (ignoreExceptions.test(t)) return;
-                                                    super.onUncaughtException(t);
-                                                }
-                                            },
-                                            null,
+            @Override
+            public void onUncaughtException(Throwable t)
+            {
+                if (ignoreExceptions.test(t)) return;
+                super.onUncaughtException(t);
+            }
+        };
+        this.commandStore = new AccordCommandStore(0,
+                                                   storeService,
+                                                   agent,
+                                                   null,
                                             ignore -> new ProgressLog.NoOpProgressLog(),
                                             cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
                                             {
                                                 @Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
                                                 @Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
                                             }),
-                                            updateHolder,
-                                            journal,
-                                            new AccordCommandStore.CommandStoreExecutor(stateCache, executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + 0 + ']'), Thread.currentThread().getId()));
+                                                   updateHolder,
+                                                   journal,
+                                                   new AccordExecutorSimple(0, CommandStore.class.getSimpleName() + '[' + 0 + ']', new AccordCacheMetrics("test"), agent));
+        this.commandStore.executor().executeDirectlyWithLock(() -> {
+            commandStore.executor().setCapacity(8 << 20);
+            commandStore.executor().setWorkingSetSize(4 << 20);
+        });
+        this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
+        this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
+        var rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges(), commandStore);
+        updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
+        updateHolder.updateGlobal(topology.ranges());
+        commandStore.unsafeUpdateRangesForEpoch();
 
-        store.cache().instances().forEach(i -> {
-            i.register(new AccordStateCache.Listener()
+        shouldEvict = boolSource(rs.fork());
+        {
+            // tests used to take 1m but after many changes in accord they now take many minutes and its due to flush... so lower the frequency of flushing
+            RandomSource fork = rs.fork();
+            shouldFlush = () -> fork.decide(.01);
+        }
+        shouldCompact = boolSource(rs.fork());
+
+        commandStore.executor().cacheUnsafe().types().forEach(i -> {
+            updateLoadFunction(i, loadFunctionWrapper);
+            i.register(new AccordCache.Listener()
             {
                 @Override
-                public void onAdd(AccordCachingState state)
-                {
-                }
-
-                @Override
-                public void onRelease(AccordCachingState state)
-                {
-                }
-
-                @Override
-                public void onEvict(AccordCachingState state)
+                public void onEvict(AccordCacheEntry state)
                 {
                     evictions.add(i + " evicted " + state);
                 }
             });
         });
+    }
 
-        this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
-        this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
-        var rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges(), store);
-        store.unsafeSetRangesForEpoch(rangesForEpoch);
-        updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
-        updateHolder.updateGlobal(topology.ranges());
-
-        shouldEvict = boolSource(rs.fork());
-        {
-            // tests used to take 1m but after many changes in accord they now take many minutes and its due to flush... so lower the frequency of flushing
-            var fork = rs.fork();
-            shouldFlush = () -> fork.decide(.01);
-        }
-        shouldCompact = boolSource(rs.fork());
+    private <K, V> void updateLoadFunction(AccordCache.Type<K, V, ?> i, FunctionWrapper wrapper)
+    {
+        i.unsafeSetLoadFunction(wrapper.wrap(i.unsafeGetLoadFunction()));
     }
 
     private static BooleanSupplier boolSource(RandomSource rs)
@@ -257,29 +278,32 @@ public class SimulatedAccordCommandStore implements AutoCloseable
 
     public void maybeCacheEvict(Unseekables<RoutingKey> keys, Ranges ranges)
     {
-        AccordStateCache cache = store.cache();
-        cache.forEach(state -> {
-            Class<?> keyType = state.key().getClass();
-            if (TxnId.class.equals(keyType))
-            {
-                Command command = (Command) state.state().get();
-                if (command != null && command.known().definition.isKnown()
-                    && (command.partialTxn().keys().intersects(keys) || ranges.intersects(command.partialTxn().keys()))
-                    && shouldEvict.getAsBoolean())
-                    cache.maybeEvict(state);
-            }
-            else if (RoutableKey.class.isAssignableFrom(keyType))
-            {
-                RoutableKey key = (RoutableKey) state.key();
-                if ((keys.contains(key) || ranges.intersects(key))
-                    && shouldEvict.getAsBoolean())
-                    cache.maybeEvict(state);
-            }
-            else
-            {
-                throw new AssertionError("Unexpected key type: " + state.key().getClass());
-            }
-        });
+        try (AccordExecutor.ExclusiveGlobalCaches caches = commandStore.executor().lockCaches())
+        {
+            AccordCache cache = caches.global;
+            cache.evictionQueue().forEach(state -> {
+                Class<?> keyType = state.key().getClass();
+                if (TxnId.class.equals(keyType))
+                {
+                    Command command = (Command) state.getExclusive();
+                    if (command != null && command.known().definition.isKnown()
+                        && (command.partialTxn().keys().intersects(keys) || ranges.intersects(command.partialTxn().keys()))
+                        && shouldEvict.getAsBoolean())
+                        cache.tryEvict(state);
+                }
+                else if (RoutableKey.class.isAssignableFrom(keyType))
+                {
+                    RoutableKey key = (RoutableKey) state.key();
+                    if ((keys.contains(key) || ranges.intersects(key))
+                        && shouldEvict.getAsBoolean())
+                        cache.tryEvict(state);
+                }
+                else
+                {
+                    throw new AssertionError("Unexpected key type: " + state.key().getClass());
+                }
+            });
+        }
 
         for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
         {
@@ -334,6 +358,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
     {
         if (Thread.interrupted())
             failures.add(new InterruptedException());
+        failures.removeIf(f -> f instanceof CancellationException || f instanceof SimulatedFault);
         if (failures.isEmpty()) return;
         AssertionError error = new AssertionError("Unexpected exceptions found");
         failures.forEach(error::addSuppressed);
@@ -341,26 +366,26 @@ public class SimulatedAccordCommandStore implements AutoCloseable
         throw error;
     }
 
-    public <T> T process(TxnRequest<T> request) throws ExecutionException, InterruptedException
+    public <T extends Reply> T process(TxnRequest<T> request) throws ExecutionException, InterruptedException
     {
         return process(request, request::apply);
     }
 
-    public <T> T process(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function) throws ExecutionException, InterruptedException
+    public <T extends Reply> T process(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function) throws ExecutionException, InterruptedException
     {
         var result = processAsync(loadCtx, function);
         processAll();
         return AsyncChains.getBlocking(result);
     }
 
-    public <T> AsyncResult<T> processAsync(TxnRequest<T> request)
+    public <T extends Reply> AsyncResult<T> processAsync(TxnRequest<T> request)
     {
         return processAsync(request, request::apply);
     }
 
-    public <T> AsyncResult<T> processAsync(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
+    public <T extends Reply> AsyncResult<T> processAsync(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
     {
-        return store.submit(loadCtx, function).beginAsResult();
+        return commandStore.submit(loadCtx, function).beginAsResult();
     }
 
     public Pair<TxnId, AsyncResult<PreAccept.PreAcceptOk>> enqueuePreAccept(Txn txn, FullRoute<?> route)
@@ -404,6 +429,6 @@ public class SimulatedAccordCommandStore implements AutoCloseable
     @Override
     public void close() throws Exception
     {
-        store.shutdown();
+        commandStore.shutdown();
     }
 }
diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java.orig b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java.orig
new file mode 100644
index 0000000000..f48156a3b3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java.orig
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.ToLongFunction;
+
+import accord.api.LocalListeners;
+import accord.api.ProgressLog;
+import accord.api.RemoteListeners;
+import accord.api.RoutingKey;
+import accord.impl.DefaultLocalListeners;
+import accord.impl.SizeOfIntersectionSorter;
+import accord.impl.TestAgent;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.CommandStores;
+import accord.local.DurableBefore;
+import accord.local.Node;
+import accord.local.NodeCommandStoreService;
+import accord.local.TimeService;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.messages.BeginRecovery;
+import accord.messages.PreAccept;
+import accord.messages.TxnRequest;
+import accord.primitives.AbstractUnseekableKeys;
+import accord.primitives.Ballot;
+import accord.primitives.FullRoute;
+import accord.primitives.Ranges;
+import accord.primitives.Routable;
+import accord.primitives.RoutableKey;
+import accord.primitives.RoutingKeys;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.primitives.Unseekables;
+import accord.topology.Topologies;
+import accord.topology.Topology;
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.SimulatedExecutorFactory;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.metrics.AccordCacheMetrics;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Generators;
+import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
+import static org.apache.cassandra.utils.AccordGenerators.fromQT;
+
+public class SimulatedAccordCommandStore implements AutoCloseable
+{
+    private final List<Throwable> failures = new ArrayList<>();
+    private final SimulatedExecutorFactory globalExecutor;
+    private final CommandStore.EpochUpdateHolder updateHolder;
+    private final BooleanSupplier shouldEvict, shouldFlush, shouldCompact;
+
+    public final NodeCommandStoreService storeService;
+    public final AccordCommandStore store;
+    public final Node.Id nodeId;
+    public final Topology topology;
+    public final Topologies topologies;
+    public final MockJournal journal;
+    public final ScheduledExecutorPlus unorderedScheduled;
+    public final List<String> evictions = new ArrayList<>();
+    public Predicate<Throwable> ignoreExceptions = ignore -> false;
+
+    public SimulatedAccordCommandStore(RandomSource rs)
+    {
+        globalExecutor = new SimulatedExecutorFactory(rs.fork(), fromQT(Generators.TIMESTAMP_GEN.map(java.sql.Timestamp::getTime)).mapToLong(TimeUnit.MILLISECONDS::toNanos).next(rs), failures::add);
+        this.unorderedScheduled = globalExecutor.scheduled("ignored");
+        ExecutorFactory.Global.unsafeSet(globalExecutor);
+        Stage.READ.unsafeSetExecutor(unorderedScheduled);
+        Stage.MUTATION.unsafeSetExecutor(unorderedScheduled);
+        for (Stage stage : Arrays.asList(Stage.MISC, Stage.ACCORD_MIGRATION, Stage.READ, Stage.MUTATION))
+            stage.unsafeSetExecutor(globalExecutor.configureSequential("ignore").build());
+
+        this.updateHolder = new CommandStore.EpochUpdateHolder();
+        this.nodeId = AccordTopology.tcmIdToAccord(ClusterMetadata.currentNullable().myNodeId());
+        this.storeService = new NodeCommandStoreService()
+        {
+            private final ToLongFunction<TimeUnit> elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.NANOSECONDS, this::now);
+
+            @Override public DurableBefore durableBefore() { return DurableBefore.EMPTY; }
+
+            @Override
+            public Timestamp uniqueNow()
+            {
+                return uniqueNow(Timestamp.NONE);
+            }
+
+            @Override
+            public Node.Id id()
+            {
+                return nodeId;
+            }
+
+            @Override
+            public long epoch()
+            {
+                return ClusterMetadata.current().epoch.getEpoch();
+            }
+
+            @Override
+            public long now()
+            {
+                return globalExecutor.nanoTime();
+            }
+
+            @Override
+            public long elapsed(TimeUnit unit)
+            {
+                return elapsed.applyAsLong(unit);
+            }
+
+            @Override
+            public Timestamp uniqueNow(Timestamp atLeast)
+            {
+                var now = Timestamp.fromValues(epoch(), now(), nodeId);
+                if (now.compareTo(atLeast) < 0)
+                    throw new UnsupportedOperationException();
+                return now;
+            }
+        };
+
+        AccordStateCache stateCache = new AccordStateCache(Stage.READ.executor(), Stage.MUTATION.executor(), 8 << 20, new AccordStateCacheMetrics("test"));
+        this.journal = new MockJournal();
+        this.store = new AccordCommandStore(0,
+                                            storeService,
+                                            new TestAgent.RethrowAgent()
+                                            {
+                                                @Override
+                                                public long preAcceptTimeout()
+                                                {
+                                                    return Long.MAX_VALUE;
+                                                }
+
+                                                @Override
+                                                public void onUncaughtException(Throwable t)
+                                                {
+                                                    if (ignoreExceptions.test(t)) return;
+                                                    super.onUncaughtException(t);
+                                                }
+                                            },
+                                            null,
+                                            ignore -> new ProgressLog.NoOpProgressLog(),
+                                            cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
+                                            {
+                                                @Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
+                                                @Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
+                                            }),
+                                            updateHolder,
+                                            journal,
+<<<<<<< HEAD
+                                            new AccordCommandStore.CommandStoreExecutor(stateCache, executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + 0 + ']'), Thread.currentThread().getId()));
+=======
+                                            new AccordCommandStoreExecutor(new AccordStateCacheMetrics("test"), executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + 0 + ']'), agent));
+
+        this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
+        this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
+        var rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges(), store);
+        //store.unsafeSetRangesForEpoch(rangesForEpoch);
+        updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
+        updateHolder.updateGlobal(topology.ranges());
+
+        shouldEvict = boolSource(rs.fork());
+        shouldFlush = boolSource(rs.fork());
+        shouldCompact = boolSource(rs.fork());
+>>>>>>> 04671b52ef (Set ranges for epoch in AccordCommandStore via super call, not by fixing up Simulated store)
+
+        store.cache().instances().forEach(i -> {
+            i.register(new AccordStateCache.Listener()
+            {
+                @Override
+                public void onAdd(AccordCachingState state)
+                {
+                }
+
+                @Override
+                public void onRelease(AccordCachingState state)
+                {
+                }
+
+                @Override
+                public void onEvict(AccordCachingState state)
+                {
+                    evictions.add(i + " evicted " + state);
+                }
+            });
+        });
+
+        this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
+        this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
+        var rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges(), store);
+        updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
+        updateHolder.updateGlobal(topology.ranges());
+
+        shouldEvict = boolSource(rs.fork());
+        shouldFlush = boolSource(rs.fork());
+        shouldCompact = boolSource(rs.fork());
+    }
+
+    private static BooleanSupplier boolSource(RandomSource rs)
+    {
+        var gen = Gens.bools().mixedDistribution().next(rs);
+        return () -> gen.next(rs);
+    }
+
+    public TxnId nextTxnId(Txn.Kind kind, Routable.Domain domain)
+    {
+        return new TxnId(storeService.epoch(), storeService.now(), kind, domain, nodeId);
+    }
+
+    public void maybeCacheEvict(Unseekables<?> keysOrRanges)
+    {
+        switch (keysOrRanges.domain())
+        {
+            case Key:
+                maybeCacheEvict((AbstractUnseekableKeys) keysOrRanges, Ranges.EMPTY);
+                break;
+            case Range:
+                maybeCacheEvict(RoutingKeys.EMPTY, (Ranges) keysOrRanges);
+                break;
+            default:
+                throw new UnsupportedOperationException("Unknown domain: " + keysOrRanges.domain());
+        }
+    }
+
+    public void maybeCacheEvict(Unseekables<RoutingKey> keys, Ranges ranges)
+    {
+        AccordStateCache cache = store.cache();
+        cache.forEach(state -> {
+            Class<?> keyType = state.key().getClass();
+            if (TxnId.class.equals(keyType))
+            {
+                Command command = (Command) state.state().get();
+                if (command != null && command.known().definition.isKnown()
+                    && (command.partialTxn().keys().intersects(keys) || ranges.intersects(command.partialTxn().keys()))
+                    && shouldEvict.getAsBoolean())
+                    cache.maybeEvict(state);
+            }
+            else if (RoutableKey.class.isAssignableFrom(keyType))
+            {
+                RoutableKey key = (RoutableKey) state.key();
+                if ((keys.contains(key) || ranges.intersects(key))
+                    && shouldEvict.getAsBoolean())
+                    cache.maybeEvict(state);
+            }
+            else
+            {
+                throw new AssertionError("Unexpected key type: " + state.key().getClass());
+            }
+        });
+
+        for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+        {
+            Memtable memtable = store.getCurrentMemtable();
+            if (memtable.partitionCount() == 0 || !intersects(store, memtable, keys, ranges))
+                continue;
+            if (shouldFlush.getAsBoolean())
+                store.forceBlockingFlush(UNIT_TESTS);
+        }
+        for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+        {
+            if (store.getLiveSSTables().size() > 5 && shouldCompact.getAsBoolean())
+            {
+                // compaction no-op since auto-compaction is disabled... so need to enable quickly
+                store.enableAutoCompaction();
+                try
+                {
+                    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
+                }
+                finally
+                {
+                    store.disableAutoCompaction();
+                }
+            }
+        }
+    }
+
+    private static boolean intersects(ColumnFamilyStore store, Memtable memtable, Unseekables<RoutingKey> keys, Ranges ranges)
+    {
+        if (keys.isEmpty() && ranges.isEmpty()) // shouldn't happen, but just in case...
+            return false;
+        switch (store.name)
+        {
+            case "commands_for_key":
+                // pk = (store_id, routing_key)
+                // since this is simulating a single store, store_id is a constant, so check key
+                try (var it = memtable.partitionIterator(ColumnFilter.NONE, DataRange.allData(store.getPartitioner()), null))
+                {
+                    while (it.hasNext())
+                    {
+                        var key = AccordKeyspace.CommandsForKeysAccessor.getKey(it.next().partitionKey());
+                        if (keys.contains(key) || ranges.intersects(key))
+                            return true;
+                    }
+                }
+                break;
+        }
+        return false;
+    }
+
+    public void checkFailures()
+    {
+        if (Thread.interrupted())
+            failures.add(new InterruptedException());
+        if (failures.isEmpty()) return;
+        AssertionError error = new AssertionError("Unexpected exceptions found");
+        failures.forEach(error::addSuppressed);
+        failures.clear();
+        throw error;
+    }
+
+    public <T> T process(TxnRequest<T> request) throws ExecutionException, InterruptedException
+    {
+        return process(request, request::apply);
+    }
+
+    public <T> T process(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function) throws ExecutionException, InterruptedException
+    {
+        var result = processAsync(loadCtx, function);
+        processAll();
+        return AsyncChains.getBlocking(result);
+    }
+
+    public <T> AsyncResult<T> processAsync(TxnRequest<T> request)
+    {
+        return processAsync(request, request::apply);
+    }
+
+    public <T> AsyncResult<T> processAsync(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
+    {
+        return store.submit(loadCtx, function).beginAsResult();
+    }
+
+    public Pair<TxnId, AsyncResult<PreAccept.PreAcceptOk>> enqueuePreAccept(Txn txn, FullRoute<?> route)
+    {
+        TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain());
+        PreAccept preAccept = new PreAccept(nodeId, topologies, txnId, txn, route);
+        return Pair.create(txnId, processAsync(preAccept, safe -> {
+            var reply = preAccept.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            return (PreAccept.PreAcceptOk) reply;
+        }));
+    }
+
+    public Pair<TxnId, AsyncResult<BeginRecovery.RecoverOk>> enqueueBeginRecovery(Txn txn, FullRoute<?> route)
+    {
+        TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain());
+        Ballot ballot = Ballot.fromValues(storeService.epoch(), storeService.now(), nodeId);
+        BeginRecovery br = new BeginRecovery(nodeId, topologies, txnId, null, txn, route, ballot);
+
+        return Pair.create(txnId, processAsync(br, safe -> {
+            var reply = br.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            return (BeginRecovery.RecoverOk) reply;
+        }).beginAsResult());
+    }
+
+    public void processAll()
+    {
+        while (processOne())
+        {
+        }
+    }
+
+    private boolean processOne()
+    {
+        boolean result = globalExecutor.processOne();
+        checkFailures();
+        return result;
+    }
+
+    @Override
+    public void close() throws Exception
+    {
+        store.shutdown();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/service/accord/async/SimulatedAsyncOperationTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java
similarity index 71%
rename from test/unit/org/apache/cassandra/service/accord/async/SimulatedAsyncOperationTest.java
rename to test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java
index 124749025d..67a855fbae 100644
--- a/test/unit/org/apache/cassandra/service/accord/async/SimulatedAsyncOperationTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java
@@ -16,12 +16,15 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord.async;
+package org.apache.cassandra.service.accord;
 
-import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
 import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
 import java.util.function.BooleanSupplier;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -45,18 +48,14 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.SimulatedAccordCommandStore;
-import org.apache.cassandra.service.accord.SimulatedAccordCommandStoreTestBase;
-import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.SimulatedAccordCommandStore.FunctionWrapper;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.utils.Pair;
 import org.assertj.core.api.Assertions;
 
 import static accord.utils.Property.qt;
 
-public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTestBase
+public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase
 {
     @Before
     public void precondition()
@@ -68,19 +67,20 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
     @Test
     public void happyPath()
     {
-        qt().withExamples(100).check(rs -> test(rs, 100, intTbl, ignore -> Action.SUCCESS));
+        qt().withExamples(100).check(rs -> test(rs, 100, intTbl, ignore -> Action.SUCCESS, ignore -> 0L));
     }
 
     @Test
     public void fuzz()
     {
         Gen<Action> actionGen = Gens.enums().allWithWeights(Action.class, 10, 1, 1);
-        qt().withExamples(100).check(rs -> test(rs, 100, intTbl, actionGen));
+        Gen.LongGen delaysNanos = Gens.longs().between(0, TimeUnit.MILLISECONDS.toNanos(10));
+        qt().withExamples(100).check(rs -> test(rs, 100, intTbl, actionGen, delaysNanos));
     }
 
     enum Operation { Task, PreAccept }
 
-    private static void test(RandomSource rs, int numSamples, TableMetadata tbl, Gen<Action> actionGen) throws Exception
+    private static void test(RandomSource rs, int numSamples, TableMetadata tbl, Gen<Action> actionGen, Gen.LongGen delaysNanos) throws Exception
     {
         AccordKeyspace.unsafeClear();
         Gen<Operation> operationGen = Gens.enums().all(Operation.class);
@@ -95,7 +95,7 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
         Gen<Unseekables<?>> unseekablesGen = Gens.oneOf(keysGen, rangesGen);
         Gen<Pair<Txn, FullRoute<?>>> txnGen = randomTxn(mixedDomainGen.next(rs), mixedTokenGen.next(rs));
 
-        try (var instance = new SimulatedAccordCommandStore(rs))
+        try (var instance = new SimulatedAccordCommandStore(rs, new SimulatedLoadFunctionWrapper(actionGen.asSupplier(rs), delaysNanos.asLongSupplier(rs))))
         {
             instance.ignoreExceptions = t -> t instanceof SimulatedFault;
             Counter counter = new Counter();
@@ -108,7 +108,7 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
                     {
                         PreLoadContext ctx = PreLoadContext.contextFor(unseekablesGen.next(rs));
                         instance.maybeCacheEvict(ctx.keys());
-                        operation(instance, ctx, actionGen.next(rs), rs::nextBoolean).begin(counter);
+                        operation(instance, ctx, actionGen.next(rs), rs::nextBoolean).chain().begin(counter);
                     }
                     break;
                     case PreAccept:
@@ -138,8 +138,14 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
             }
             instance.processAll();
             Assertions.assertThat(counter.counter).isEqualTo(numSamples);
-            instance.store.cache().stream().forEach(e -> {
-                Assertions.assertThat(e.referenceCount()).isEqualTo(0);
+            instance.commandStore.cachesUnsafe().commands().forEach(e -> {
+                Assertions.assertThat(e.references()).isEqualTo(0);
+            });
+            instance.commandStore.cachesUnsafe().commandsForKeys().forEach(e -> {
+                Assertions.assertThat(e.references()).isEqualTo(0);
+            });
+            instance.commandStore.cachesUnsafe().timestampsForKeys().forEach(e -> {
+                Assertions.assertThat(e.references()).isEqualTo(0);
             });
         }
     }
@@ -171,18 +177,11 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
         return new TokenRange(new TokenKey(tableId, new LongToken(start)), new TokenKey(tableId, new LongToken(end)));
     }
 
-    private enum Action {SUCCESS, FAILURE, LOAD_FAILURE}
+    private enum Action { SUCCESS, FAILURE, LOAD_FAILURE }
 
-    private static AsyncOperation<Void> operation(SimulatedAccordCommandStore instance, PreLoadContext ctx, Action action, BooleanSupplier delay)
+    private static AccordTask<Void> operation(SimulatedAccordCommandStore instance, PreLoadContext ctx, Action action, BooleanSupplier delay)
     {
-        return new SimulatedOperation(instance.store, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS)
-        {
-            @Override
-            AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
-            {
-                return new SimulatedLoader(action == SimulatedAsyncOperationTest.Action.LOAD_FAILURE ? SimulatedLoader.Action.FAILURE : SimulatedLoader.Action.SUCCESS, delay.getAsBoolean(), instance.unorderedScheduled);
-            }
-        };
+        return new SimulatedOperation(instance.commandStore, ctx, action == Action.FAILURE ? SimulatedOperation.Action.FAILURE : SimulatedOperation.Action.SUCCESS);
     }
 
     private static class Counter implements BiConsumer<Object, Throwable>
@@ -198,7 +197,7 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
         }
     }
 
-    private static class SimulatedOperation extends AsyncOperation<Void>
+    private static class SimulatedOperation extends AccordTask<Void>
     {
         enum Action { SUCCESS, FAILURE}
         private final Action action;
@@ -218,37 +217,45 @@ public class SimulatedAsyncOperationTest extends SimulatedAccordCommandStoreTest
         }
     }
 
-    private static class SimulatedLoader extends AsyncLoader
+    private static class SimulatedLoadFunctionWrapper implements FunctionWrapper
     {
+        final Supplier<Action> actions;
+        final LongSupplier delayNanos;
 
-        enum Action { SUCCESS, FAILURE}
-
-        private final Action action;
-        private boolean delay;
-        private final ScheduledExecutorService executor;
-        SimulatedLoader(Action action, boolean delay, ScheduledExecutorService executor)
+        private SimulatedLoadFunctionWrapper(Supplier<Action> actions, LongSupplier delayNanos)
         {
-            super(null, null, null, null);
-            this.action = action;
-            this.delay = delay;
-            this.executor = executor;
+            this.actions = actions;
+            this.delayNanos = delayNanos;
         }
 
         @Override
-        public boolean load(TxnId primaryTxnId, AsyncOperation.Context context, BiConsumer<Object, Throwable> callback)
+        public <I1, I2, O> BiFunction<I1, I2, O> wrap(BiFunction<I1, I2, O> f)
         {
-            if (delay)
-            {
-                executor.schedule(() -> {
-                    callback.accept(null, action == Action.FAILURE ? new SimulatedFault("Failure loading " + context) : null);
-                }, 1, TimeUnit.SECONDS);
-                delay = false;
-                return false;
-            }
-            if (action == Action.FAILURE)
-                throw new SimulatedFault("Failure loading " + context);
+            return new SimulatedLoadFunction<>(f, actions, delayNanos);
+        }
+    }
 
-            return true;
+    private static class SimulatedLoadFunction<I1, I2, V> implements BiFunction<I1, I2, V>
+    {
+        private final BiFunction<I1, I2, V> load;
+        private final Supplier<Action> actions;
+        private final LongSupplier delaysNanos;
+        SimulatedLoadFunction(BiFunction<I1, I2, V> load, Supplier<Action> actions, LongSupplier delaysNanos)
+        {
+            this.load = load;
+            this.actions = actions;
+            this.delaysNanos = delaysNanos;
+        }
+
+        @Override
+        public V apply(I1 i1, I2 i2)
+        {
+            long delayNanos = delaysNanos.getAsLong();
+            if (delayNanos > 0)
+                LockSupport.parkNanos(delayNanos);
+            Action action = actions.get();
+            if (action == Action.SUCCESS) return load.apply(i1, i2);
+            throw new SimulatedFault("Failure loading " + i2);
         }
     }
 }
diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java
index 1d9935084c..770af5b314 100644
--- a/test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java
@@ -186,7 +186,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
                 FullRangeRoute rangeRoute = ranges.toRoute(pk.toUnseekable());
                 Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, ranges);
 
-                DepsModel model = new DepsModel(instance.store.unsafeRangesForEpoch().currentRanges());
+                DepsModel model = new DepsModel(instance.commandStore.unsafeRangesForEpoch().currentRanges());
                 for (int i = 0; i < numSamples; i++)
                 {
                     instance.maybeCacheEvict(keyRoute, ranges);
@@ -258,7 +258,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
                 Range left = tokenRange(tbl.id, token - 10, token + 5);
                 Range right = tokenRange(tbl.id, token - 5, token + 10);
 
-                DepsModel model = new DepsModel(instance.store.unsafeRangesForEpoch().currentRanges());
+                DepsModel model = new DepsModel(instance.commandStore.unsafeRangesForEpoch().currentRanges());
                 for (int i = 0; i < numSamples; i++)
                 {
                     Ranges partialRange = Ranges.of(rs.nextBoolean() ? left : right);
diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java
index 16a4d83065..cf09d8eab8 100644
--- a/test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java
@@ -71,7 +71,7 @@ public class SimulatedMultiKeyAndRangeTest extends SimulatedAccordCommandStoreTe
                 Gen.IntGen keyCountGen = keyDistribution.next(rs);
                 Gen.IntGen rangeCountGen = rangeDistribution.next(rs);
 
-                DepsModel model = new DepsModel(instance.store.unsafeRangesForEpoch().currentRanges());
+                DepsModel model = new DepsModel(instance.commandStore.unsafeRangesForEpoch().currentRanges());
 
                 for (int i = 0; i < numSamples; i++)
                 {
diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java b/test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java
index d1edbb4455..68a2b3cef3 100644
--- a/test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java
+++ b/test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java
@@ -87,7 +87,8 @@ public class SimulatedRandomKeysWithRangeConflictTest extends SimulatedAccordCom
         {
             AccordKeyspace.unsafeClear();
             this.instance = new SimulatedAccordCommandStore(rs);
-            this.model = new DepsModel(instance.store.unsafeRangesForEpoch().currentRanges());
+            this.instance.commandStore.executor().cacheUnsafe().setShrinkingOn(false);
+            this.model = new DepsModel(instance.commandStore.unsafeRangesForEpoch().currentRanges());
         }
 
         @Override
diff --git a/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java b/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java
deleted file mode 100644
index 8c893c4a21..0000000000
--- a/test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java
+++ /dev/null
@@ -1,455 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord.async;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import accord.api.RoutingKey;
-import accord.local.cfk.CommandsForKey;
-import accord.impl.TimestampsForKey;
-import accord.local.Command;
-import accord.local.KeyHistory;
-import accord.primitives.PartialTxn;
-import accord.primitives.RoutingKeys;
-import accord.primitives.TxnId;
-import accord.utils.async.AsyncChains;
-import accord.utils.async.AsyncResult;
-import accord.utils.async.AsyncResults;
-import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.concurrent.ExecutorFactory;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.concurrent.ManualExecutor;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.AccordCachingState;
-import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeState;
-import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
-import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.service.accord.async.AsyncOperation.Context;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
-import org.apache.cassandra.utils.concurrent.Condition;
-
-import static accord.local.KeyHistory.COMMANDS;
-import static accord.local.KeyHistory.TIMESTAMPS;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singleton;
-import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
-import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.notDefined;
-import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted;
-import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore;
-import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn;
-import static org.apache.cassandra.service.accord.AccordTestUtils.execute;
-import static org.apache.cassandra.service.accord.AccordTestUtils.loaded;
-import static org.apache.cassandra.service.accord.AccordTestUtils.testLoad;
-import static org.apache.cassandra.service.accord.AccordTestUtils.txnId;
-
-public class AsyncLoaderTest
-{
-    @BeforeClass
-    public static void beforeClass() throws Throwable
-    {
-        SchemaLoader.prepareServer();
-        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1),
-                                    parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c)) WITH transactional_mode='full'", "ks"));
-        StorageService.instance.initServer();
-    }
-
-    /**
-     * Loading a cached resource shouldn't block
-     */
-    @Test
-    public void cachedTest()
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ManualExecutor executor = new ManualExecutor();
-        AccordCommandStore commandStore =
-            createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-        AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
-        commandStore.executeBlocking(() -> commandStore.cache().setCapacity(1024));
-
-        AccordStateCache.Instance<RoutingKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-        TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
-
-        // acquire / release
-
-        commandCache.unsafeSetLoadFunction(id -> notDefined(id, txn));
-        AccordSafeCommand safeCommand = commandCache.acquire(txnId);
-        testLoad(executor, safeCommand, notDefined(txnId, txn));
-        AccordCachingState<TxnId, Command> safeCommandGlobal = safeCommand.global();
-        commandCache.release(safeCommand);
-
-        timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((TokenKey) k));
-        AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquire(key);
-        testLoad(executor, safeTimestamps, new TimestampsForKey(key));
-        AccordCachingState<RoutingKey, TimestampsForKey> safeTimestampsGlobal = safeTimestamps.global();
-        timestampsCache.release(safeTimestamps);
-
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), RoutingKeys.of(key), TIMESTAMPS);
-
-        // everything is cached, so the loader should return immediately
-        commandStore.executeBlocking(() -> {
-            Context context = new Context();
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertEquals(safeCommandGlobal, context.commands.get(txnId).global());
-            Assert.assertEquals(safeTimestampsGlobal, context.timestampsForKey.get(key).global());
-            Assert.assertTrue(result);
-        });
-
-        Assert.assertSame(safeCommandGlobal, commandCache.getUnsafe(txnId));
-        Assert.assertSame(safeTimestampsGlobal, timestampsCache.getUnsafe(key));
-    }
-
-    /**
-     * Loading a cached resource should block
-     */
-    @Test
-    public void loadTest()
-    {
-        AtomicLong clock = new AtomicLong(0);
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-        TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
-
-        // create / persist
-        AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null));
-        safeCommand.preExecute();
-        safeCommand.set(notDefined(txnId, txn));
-        AccordKeyspace.getCommandMutation(commandStore, safeCommand, commandStore.nextSystemTimestampMicros()).apply();
-
-        AccordSafeTimestampsForKey timestamps = new AccordSafeTimestampsForKey(loaded(key, null));
-        timestamps.preExecute();
-        timestamps.initialize();
-
-        AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), timestamps.current(), commandStore.nextSystemTimestampMicros()).apply();
-
-        // resources are on disk only, so the loader should suspend...
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), RoutingKeys.of(key), TIMESTAMPS);
-        AsyncPromise<Void> cbFired = new AsyncPromise<>();
-        Context context = new Context();
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> {
-                Assert.assertNull(t);
-                Assert.assertTrue(context.commands.containsKey(txnId));
-                Assert.assertTrue(context.timestampsForKey.containsKey(key));
-                cbFired.setSuccess(null);
-            });
-            Assert.assertFalse(result);
-        });
-
-        cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-
-        // then return immediately after the callback has fired
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertTrue(context.commands.containsKey(txnId));
-            Assert.assertTrue(context.timestampsForKey.containsKey(key));
-            Assert.assertTrue(result);
-        });
-    }
-
-    /**
-     * Test when some resources are cached and others need to be loaded
-     */
-    @Test
-    public void partialLoadTest()
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ManualExecutor executor = new ManualExecutor();
-        AccordCommandStore commandStore =
-            createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-        AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-        TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
-
-        // acquire /release, create / persist
-        commandCache.unsafeSetLoadFunction(id -> notDefined(id, txn));
-        AccordSafeCommand safeCommand = commandCache.acquire(txnId);
-        testLoad(executor, safeCommand, notDefined(txnId, txn));
-        commandCache.release(safeCommand);
-
-        AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), new TimestampsForKey(key), commandStore.nextSystemTimestampMicros()).apply();
-
-        // resources are on disk only, so the loader should suspend...
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), RoutingKeys.of(key), TIMESTAMPS);
-        AsyncPromise<Void> cbFired = new AsyncPromise<>();
-        Context context = new Context();
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> {
-                Assert.assertNull(t);
-                Assert.assertTrue(context.commands.containsKey(txnId));
-                Assert.assertTrue(context.timestampsForKey.containsKey(key));
-                cbFired.setSuccess(null);
-            });
-            Assert.assertFalse(result);
-        });
-
-        executor.runOne();
-        cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-
-        // then return immediately after the callback has fired
-        commandStore.executeBlocking(() -> {
-
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertTrue(context.commands.containsKey(txnId));
-            Assert.assertTrue(context.timestampsForKey.containsKey(key));
-            Assert.assertTrue(result);
-        });
-    }
-
-    /**
-     * If another process is loading a resource, piggyback on it's future
-     */
-    @Test
-    public void inProgressLoadTest() throws Throwable
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ManualExecutor executor = new ManualExecutor();
-        AccordCommandStore commandStore =
-            createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-        commandStore.executor().submit(() -> commandStore.cache().setCapacity(1024)).get();
-        AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-        TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
-
-        commandCache.unsafeSetLoadFunction(id -> { Assert.assertEquals(txnId, id); return notDefined(id, txn); });
-        AccordSafeCommand safeCommand = commandCache.acquire(txnId);
-        Assert.assertEquals(AccordCachingState.Status.LOADING, safeCommand.globalStatus());
-        Assert.assertTrue(commandCache.isReferenced(txnId));
-        Assert.assertFalse(commandCache.isLoaded(txnId));
-
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), RoutingKeys.of(key), KeyHistory.NONE);
-
-        // since there's a read future associated with the txnId, we'll wait for it to load
-        AsyncPromise<Void> cbFired = new AsyncPromise<>();
-        Context context = new Context();
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> {
-                Assert.assertNull(t);
-                Assert.assertTrue(context.commands.containsKey(txnId));
-                Assert.assertFalse(context.timestampsForKey.containsKey(key));
-                cbFired.setSuccess(null);
-            });
-            Assert.assertFalse(result);
-        });
-
-        Assert.assertFalse(cbFired.isSuccess());
-        executor.runOne();
-        Assert.assertEquals(AccordCachingState.Status.LOADED, safeCommand.globalStatus());
-        cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-        Assert.assertTrue(cbFired.isSuccess());
-
-        // then return immediately after the callback has fired
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertTrue(context.commands.containsKey(txnId));
-            Assert.assertFalse(context.timestampsForKey.containsKey(key));
-            Assert.assertTrue(result);
-        });
-    }
-
-    @Test
-    public void failedLoadTest() throws Throwable
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ExecutorPlus executor = ExecutorFactory.Global.executorFactory().sequential("GlobalLogFollower");
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-
-        TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1);
-        TxnId txnId2 = txnId(1, clock.incrementAndGet(), 1);
-
-        AsyncResult.Settable<Void> promise = AsyncResults.settable();
-        AsyncResult.Settable<Void> callback = AsyncResults.settable();
-        RuntimeException failure = new RuntimeException();
-
-        Condition startResponding = Condition.newOneTimeCondition();
-        Condition loadedAll = Condition.newOneTimeCondition();
-        execute(commandStore, () -> {
-            AtomicInteger loadCalls = new AtomicInteger();
-
-            commandStore.commandCache().unsafeSetLoadFunction(txnId ->
-            {
-                startResponding.awaitUninterruptibly();
-                loadCalls.incrementAndGet();
-
-                if (!txnId.equals(txnId1) && !txnId.equals(txnId2))
-                    throw new AssertionError("Unknown txnId: " + txnId);
-
-                if (loadCalls.get() == 2)
-                {
-                    loadedAll.signal();
-                    throw failure;
-                }
-
-                return notDefined(txnId, null);
-            });
-
-            AsyncLoader loader = new AsyncLoader(commandStore, ImmutableList.of(txnId1, txnId2), RoutingKeys.EMPTY, KeyHistory.COMMANDS);
-
-            boolean result =  loader.load(txnId1, new Context(), (u, t) -> {
-                Assert.assertFalse(callback.isDone());
-                Assert.assertNull(u);
-                Assert.assertEquals(failure, t);
-                callback.trySuccess(null);
-            });
-            startResponding.signal();
-            loadedAll.awaitUninterruptibly();
-            Assert.assertFalse(result);
-            Assert.assertEquals(2, loadCalls.get());
-        });
-
-        promise.tryFailure(failure);
-        AsyncChains.getUninterruptibly(callback);
-    }
-
-    @Test
-    public void inProgressCommandSaveTest()
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ManualExecutor executor = new ManualExecutor();
-        AccordCommandStore commandStore =
-        createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-        AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
-
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-
-        // acquire / release
-
-        commandCache.unsafeSetLoadFunction(id -> notDefined(id, txn));
-        commandCache.unsafeSetSaveFunction((after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
-
-        AccordSafeCommand safeCommand = commandCache.acquire(txnId);
-        testLoad(executor, safeCommand, notDefined(txnId, txn));
-        safeCommand.set(preaccepted(txnId, txn, safeCommand.txnId()));
-        commandCache.release(safeCommand);
-
-        Assert.assertEquals(AccordCachingState.Status.MODIFIED, commandCache.getUnsafe(txnId).status());
-        commandCache.getUnsafe(txnId).save(executor, (after) -> () -> {});
-        Assert.assertEquals(AccordCachingState.Status.SAVING, commandCache.getUnsafe(txnId).status());
-
-        // since the command is still saving, the loader shouldn't be able to acquire a reference
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), RoutingKeys.of(), KeyHistory.NONE);
-        AsyncPromise<Void> cbFired = new AsyncPromise<>();
-        Context context = new Context();
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> {
-                Assert.assertNull(t);
-                Assert.assertTrue(context.commands.containsKey(txnId));
-                cbFired.setSuccess(null);
-            });
-            Assert.assertFalse(result);
-        });
-
-        Assert.assertEquals(AccordCachingState.Status.SAVING, commandCache.getUnsafe(txnId).status());
-        executor.runOne();
-        cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandCache.getUnsafe(txnId).status());
-
-        // then return immediately after the callback has fired
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertTrue(context.commands.containsKey(txnId));
-            Assert.assertTrue(result);
-        });
-    }
-
-    @Test
-    public void inProgressCFKSaveTest()
-    {
-        this.inProgressCFKSaveTest(COMMANDS, AccordCommandStore::commandsForKeyCache, context -> context.commandsForKey, CommandsForKey::new, (cfk, u) -> cfk.update(u).cfk());
-    }
-
-    @Test
-    public void inProgressTFKSaveTest()
-    {
-        inProgressCFKSaveTest(TIMESTAMPS, AccordCommandStore::timestampsForKeyCache, context -> context.timestampsForKey, TimestampsForKey::new, (tfk, c) -> new TimestampsForKey(tfk.key(), c.executeAt(), c.executeAt().hlc(), c.txnId(), c.executeAt()));
-    }
-
-    private <T1, T2 extends AccordSafeState<RoutingKey, T1>, C extends AccordStateCache.Instance<RoutingKey, T1, T2>>  void inProgressCFKSaveTest(KeyHistory history, Function<AccordCommandStore, C> getter, Function<Context, Map<?, ?>> inContext, Function<RoutingKey, T1> initialiser, BiFunction<T1, Command, T1> update)
-    {
-        AtomicLong clock = new AtomicLong(0);
-        ManualExecutor executor = new ManualExecutor();
-        AccordCommandStore commandStore =
-        createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
-
-        C cache = getter.apply(commandStore);
-        cache.unsafeSetSaveFunction((after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
-
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-        PartialTxn txn = createPartialTxn(0);
-        TokenKey key = ((PartitionKey) Iterables.getOnlyElement(txn.keys())).toUnseekable();
-        Command preaccepted = preaccepted(txnId, txn, txnId);
-
-        // acquire / release
-        T2 safe = cache.acquireOrInitialize(key, k -> initialiser.apply(k));
-        safe.preExecute();
-        safe.set(update.apply(safe.current(), preaccepted));
-        cache.release(safe);
-
-        Assert.assertEquals(AccordCachingState.Status.MODIFIED, cache.getUnsafe(key).status());
-        cache.getUnsafe(key).save(executor, (after) -> () -> {});
-        Assert.assertEquals(AccordCachingState.Status.SAVING, cache.getUnsafe(key).status());
-
-        // since the command is still saving, the loader shouldn't be able to acquire a reference
-        AsyncLoader loader = new AsyncLoader(commandStore, emptyList(), RoutingKeys.of(key), history);
-        AsyncPromise<Void> cbFired = new AsyncPromise<>();
-        Context context = new Context();
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> {
-                Assert.assertNull(t);
-                Assert.assertEquals(context.timestampsForKey.containsKey(key), inContext.apply(context) == context.timestampsForKey);
-                Assert.assertEquals(context.commandsForKey.containsKey(key), inContext.apply(context) == context.commandsForKey);
-                cbFired.setSuccess(null);
-            });
-            Assert.assertFalse(result);
-        });
-
-        executor.runOne();
-        cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-
-        // then return immediately after the callback has fired
-        commandStore.executeBlocking(() -> {
-            boolean result = loader.load(txnId, context, (o, t) -> Assert.fail());
-            Assert.assertEquals(context.timestampsForKey.containsKey(key), inContext.apply(context) == context.timestampsForKey);
-            Assert.assertEquals(context.commandsForKey.containsKey(key), inContext.apply(context) == context.commandsForKey);
-            Assert.assertTrue(result);
-        });
-    }
-}
diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java
index 28ae575fb2..99097c064c 100644
--- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java
@@ -450,6 +450,7 @@ public class AccordGenerators
     {
         return rs -> {
             Range range = rangeGen.next(rs);
+            TxnId locallyWitnessedOrInvalidatedBefore = emptyGen.next(rs) ? TxnId.NONE : txnIdGen.next(rs); // emptyable or range
             TxnId locallyAppliedOrInvalidatedBefore = emptyGen.next(rs) ? TxnId.NONE : txnIdGen.next(rs); // emptyable or range
             TxnId locallyDecidedAndAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
             TxnId shardAppliedOrInvalidatedBefore = emptyGen.next(rs) ? TxnId.NONE : txnIdGen.next(rs); // emptyable or range
@@ -461,7 +462,7 @@ public class AccordGenerators
             long maxEpoch = Stream.of(locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast).filter(t -> t != null).mapToLong(Timestamp::epoch).max().getAsLong();
             long startEpoch = rs.nextLong(maxEpoch);
             long endEpoch = emptyGen.next(rs) ? Long.MAX_VALUE : 1 + rs.nextLong(startEpoch, Long.MAX_VALUE);
-            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
+            return new RedundantBefore.Entry(range, startEpoch, endEpoch, locallyWitnessedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardOnlyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         };
     }