Merge branch 'cassandra-6.0' into trunk

modules/accord

@@ -1 +1 @@
-Subproject commit 8229cbf269f14553c338cf47efc9466bbc7ee81b
+Subproject commit d3c5935589082f784b10dbee4d89691612f680eb

src/java/org/apache/cassandra/config/AccordSpec.java

@@ -28,18 +28,19 @@ import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.service.accord.serializers.Version;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 
+import static org.apache.cassandra.config.AccordSpec.CatchupMode.NORMAL;
 import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_POOL_PER_SHARD;
 import static org.apache.cassandra.config.AccordSpec.QueueSubmissionModel.SYNC;
 import static org.apache.cassandra.config.AccordSpec.RangeIndexMode.in_memory;
 
+// TODO (expected): rename to AccordConf?
 public class AccordSpec
 {
     public volatile boolean enabled = false;
 
+    // TODO (expected): move to JournalSpec
     public volatile String journal_directory;
 
-    public volatile boolean enable_journal_compaction = true;
-
     /**
      * Enables the virtual Accord debug-only keyspace with tables
      * that expose internal state to aid the developers working
@@ -92,6 +93,8 @@ public class AccordSpec
 
         /**
          * The queue workers only require ownership of the lock, submissions happens fully asynchronously.
+         *
+         * NOTE: EXPERIMENTAL
          */
         ASYNC,
 
@@ -136,16 +139,16 @@ public class AccordSpec
     public DurationSpec.IntMillisecondsBound repair_timeout = new DurationSpec.IntMillisecondsBound("10m");
     public String recover_txn = "5s*attempts <= 60s";
     public StringRetryStrategy recover_syncpoint = new StringRetryStrategy("60s <= 30s*attempts...60s*attempts <= 600s");
-    public String fetch_txn = "1s*attempts";
-    public String fetch_syncpoint = "5s*attempts";
-    public String expire_txn = "5s*attempts";
+    public String fetch_txn = "2s*attempts <= 60s";
+    public String fetch_syncpoint = "5s*attempts <= 60s";
+    public String expire_txn = "5s*attempts <= 60s";
     public String expire_syncpoint = "60s*attempts<=300s";
     public String expire_epoch_wait = "10s";
     // we don't want to wait ages for durability as it blocks other durability progress; even this might be too long, as we can always retry
     public String expire_durability = "10s*attempts <= 30s";
     public String slow_syncpoint_preaccept = "10s";
-    public String slow_txn_preaccept = "30ms <= p50*2 <= 100ms";
-    public String slow_read = "30ms <= p50*2 <= 100ms";
+    public String slow_txn_preaccept = "30ms <= p50*2 <= 1000ms";
+    public String slow_read = "30ms <= p50*2 <= 1000ms";
     public StringRetryStrategy retry_syncpoint = new StringRetryStrategy("10s*attempt <= 600s");
     public StringRetryStrategy retry_durability = new StringRetryStrategy("10s*attempt <= 600s");
     public StringRetryStrategy retry_bootstrap = new StringRetryStrategy("10s*attempt <= 600s");
@@ -156,8 +159,8 @@ public class AccordSpec
 
     public volatile DurationSpec.IntSecondsBound fast_path_update_delay = null;
 
-    public volatile int shard_durability_target_splits = 16;
-    public volatile int shard_durability_max_splits = 128;
+    public volatile int shard_durability_target_splits = 8;
+    public volatile int shard_durability_max_splits = 64;
     public volatile DurationSpec.IntSecondsBound durability_txnid_lag = new DurationSpec.IntSecondsBound(5);
     public volatile DurationSpec.IntSecondsBound shard_durability_cycle = new DurationSpec.IntSecondsBound(5, TimeUnit.MINUTES);
     public volatile DurationSpec.IntSecondsBound global_durability_cycle = new DurationSpec.IntSecondsBound(5, TimeUnit.MINUTES);
@@ -175,19 +178,39 @@ public class AccordSpec
      */
     public volatile TransactionalRangeMigration range_migration = TransactionalRangeMigration.auto;
 
+    public enum RebootstrapMode
+    {
+        full_repair, truncate_and_stream
+    }
+
+    public enum CatchupMode
+    {
+        DISABLED,
+        NORMAL,
+        FALLBACK_TO_HARD,
+        HARD
+    }
+
     /**
      * default transactional mode for tables created by this node when no transactional mode has been specified in the DDL
      */
     public TransactionalMode default_transactional_mode = TransactionalMode.off;
     public boolean ephemeralReadEnabled = true;
-    public boolean state_cache_listener_jfr_enabled = true;
+    public boolean state_cache_listener_jfr_enabled = false;
+
+    public float hard_reject_ratio = 0.5f;
+    public int min_soft_reject_count = 10;
+    public int max_soft_reject_count = 100;
+    public DurationSpec.LongMicrosecondsBound soft_reject_age = new DurationSpec.LongMicrosecondsBound("10s");
+    public DurationSpec.LongMicrosecondsBound soft_reject_cumulative_age = new DurationSpec.LongMicrosecondsBound("60s");
 
     public DurationSpec.IntSecondsBound catchup_on_start_success_latency = new DurationSpec.IntSecondsBound(60);
     public DurationSpec.IntSecondsBound catchup_on_start_fail_latency = new DurationSpec.IntSecondsBound(900);
     public int catchup_on_start_max_attempts = 5;
     // TODO (required): roll this back to catchup_on_start_exit_on_failure: true
     public boolean catchup_on_start_exit_on_failure = false;
-    public boolean catchup_on_start = true;
+    public CatchupMode catchup_on_start = NORMAL;
+    public DurationSpec.IntSecondsBound shutdown_grace_period = new DurationSpec.IntSecondsBound(15 * 60);
 
     public enum RangeIndexMode { in_memory, journal_sai }
     public RangeIndexMode range_index_mode = in_memory;
@@ -203,16 +226,68 @@ public class AccordSpec
 
     public static class JournalSpec implements Params
     {
+        public enum ReplayMode
+        {
+            /**
+             * Replay all journal entries and erase local state such as CommandsForKey that can be recreated
+             */
+            RESET,
+
+            /**
+             * Replay all journal entries
+             */
+            ALL,
+
+            /**
+             * Replay journal entries for commands that intersect a non-durable range.
+             * Ordinarily it should be necessary to only replay commands that do not intersect any durable ranges.
+             */
+            PART_NON_DURABLE,
+
+            /**
+             * Replay journal entries for commands that are not durable to the data or command stores.
+             * THIS MODE IS NOT YET SAFE TO RUN
+             */
+            NON_DURABLE
+        }
+
+        public enum ReplaySavePoint
+        {
+            NO,
+            LATEST
+        }
+
+        public enum StopMarkerFailurePolicy
+        {
+            /**
+             * If the start marker exceeds the stop marker then exit, since we cannot guarantee our consensus log is complete.
+             */
+            EXIT,
+
+            /**
+             * If the start marker exceeds the stop marker startup, assuming the consensus log has been determined complete externally.
+             * Note this is VERY UNSAFE if you care about isolation guarantees.
+             */
+            UNSAFE_STARTUP,
+
+            REBOOTSTRAP
+        }
+
         public int segmentSize = 32 << 20;
         public int compactMaxSegments = 32;
         public FailurePolicy failurePolicy = FailurePolicy.STOP;
-        public ReplayMode replayMode = ReplayMode.ONLY_NON_DURABLE;
+        public ReplayMode replay = ReplayMode.PART_NON_DURABLE;
+        public ReplaySavePoint replaySavePoint = ReplaySavePoint.LATEST;
+        public int retainSavePoints = 2;
+        public StopMarkerFailurePolicy stopMarkerFailurePolicy = StopMarkerFailurePolicy.EXIT;
+        public RecoverableCrcFailurePolicy crcFailureOnRebuildPolicy = RecoverableCrcFailurePolicy.FAIL;
         public FlushMode flushMode = FlushMode.PERIODIC;
         public volatile DurationSpec flushPeriod; // pulls default from 'commitlog_sync_period'
         public DurationSpec periodicFlushLagBlock = new DurationSpec.IntMillisecondsBound("1500ms");
         public DurationSpec.IntMillisecondsBound compactionPeriod = new DurationSpec.IntMillisecondsBound("60000ms");
         private volatile long flushCombinedBlockPeriod = Long.MIN_VALUE;
         public Version version = Version.DOWNGRADE_SAFE_VERSION;
+        public boolean enable_compaction = true;
 
         public JournalSpec setFlushPeriod(DurationSpec newFlushPeriod)
         {
@@ -246,9 +321,9 @@ public class AccordSpec
         }
 
         @Override
-        public ReplayMode replayMode()
+        public RecoverableCrcFailurePolicy crcFailureOnRebuildPolicy()
         {
-            return replayMode;
+            return crcFailureOnRebuildPolicy;
         }
 
         @Override
@@ -260,7 +335,7 @@ public class AccordSpec
         @Override
         public boolean enableCompaction()
         {
-            return DatabaseDescriptor.getAccord().enable_journal_compaction;
+            return enable_compaction;
         }
 
         @Override

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -5689,8 +5689,7 @@ public class DatabaseDescriptor
             case THREAD_PER_SHARD_SYNC_QUEUE:
                 return conf.accord.queue_shard_count.or(DatabaseDescriptor::getAvailableProcessors);
             case THREAD_POOL_PER_SHARD:
-                int defaultMax = getAccordQueueSubmissionModel() == AccordSpec.QueueSubmissionModel.SYNC ? 8 : 4;
-                return conf.accord.queue_shard_count.or(Math.min(defaultMax, DatabaseDescriptor.getAvailableProcessors()));
+                return conf.accord.queue_shard_count.or(DatabaseDescriptor.getAvailableProcessors()/4);
         }
     }
 

src/java/org/apache/cassandra/cql3/conditions/ColumnCondition.java

@@ -57,7 +57,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.invalidRequest;
 import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
-import static org.apache.cassandra.service.accord.AccordSerializers.columnMetadataSerializer;
+import static org.apache.cassandra.service.accord.serializers.AccordSerializers.columnMetadataSerializer;
 import static org.apache.cassandra.utils.ByteBufferUtil.nullableByteBufferSerializer;
 
 /**

src/java/org/apache/cassandra/cql3/statements/schema/AlterSchemaStatement.java

@@ -45,7 +45,7 @@ import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.ClientWarn;
 import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.service.accord.AccordTopology;
+import org.apache.cassandra.service.accord.topology.AccordTopology;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.Event.SchemaChange;

src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java

@@ -30,7 +30,7 @@ import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.KeyspaceParams.Option;
 import org.apache.cassandra.schema.ReplicationParams;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 
 public final class KeyspaceAttributes extends PropertyDefinitions
 {

src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java

@@ -37,7 +37,7 @@ import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.schema.TableParams.Option;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -408,8 +408,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     {
         return () -> {
             for (Keyspace keyspace : Keyspace.all())
+            {
                 for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+                {
+                    if (SchemaConstants.ACCORD_KEYSPACE_NAME.equals(cfs.keyspace.getName()))
+                        continue;
+
                     CompactionManager.instance.submitBackground(cfs);
+                }
+            }
+
         };
     }
 
@@ -1144,6 +1152,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         return postFlushExecutor.submit(current::getCommitLogLowerBound);
     }
 
+    public Future<Void> waitForPriorFlushes()
+    {
+        return postFlushExecutor.submit(() -> null);
+    }
+
     public CommitLogPosition forceBlockingFlush(FlushReason reason)
     {
         return FBUtilities.waitOnFuture(forceFlush(reason));

src/java/org/apache/cassandra/db/compaction/CompactionIterator.java

@@ -89,9 +89,6 @@ import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.accord.AccordJournal;
-import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightImage;
-import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightSerializer;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
 import org.apache.cassandra.service.accord.AccordService;
@@ -100,8 +97,14 @@ import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo;
 import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfos;
 import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.api.TokenKey;
-import org.apache.cassandra.service.accord.journal.AccordTopologyUpdate;
-import org.apache.cassandra.service.accord.journal.AccordTopologyUpdate.TopologyImage;
+import org.apache.cassandra.service.accord.journal.CommandChangeWriter;
+import org.apache.cassandra.service.accord.journal.CommandChanges;
+import org.apache.cassandra.service.accord.journal.MergeSerializer;
+import org.apache.cassandra.service.accord.journal.MergeSerializers;
+import org.apache.cassandra.service.accord.journal.MergeSerializers.CommandChangeSerializer;
+import org.apache.cassandra.service.accord.journal.Merger;
+import org.apache.cassandra.service.accord.journal.TopologyRecord;
+import org.apache.cassandra.service.accord.journal.TopologyRecord.TopologyImage;
 import org.apache.cassandra.service.accord.serializers.Version;
 import org.apache.cassandra.service.paxos.PaxosRepairHistory;
 import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
@@ -122,8 +125,9 @@ import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
 import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
 import static org.apache.cassandra.service.accord.AccordKeyspace.CFKAccessor;
 import static org.apache.cassandra.service.accord.AccordKeyspace.JournalColumns.getJournalKey;
-import static org.apache.cassandra.service.accord.journal.AccordTopologyUpdate.Kind.Image;
-import static org.apache.cassandra.service.accord.journal.AccordTopologyUpdate.Kind.Repeat;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.TopologySerializer.INSTANCE;
+import static org.apache.cassandra.service.accord.journal.TopologyRecord.Kind.Image;
+import static org.apache.cassandra.service.accord.journal.TopologyRecord.Kind.Repeat;
 
 /**
  * Merge multiple iterators over the content of sstable into a "compacted" iterator.
@@ -875,7 +879,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         final ColumnMetadata versionColumn;
 
         JournalKey key;
-        AccordRowCompactor<?> compactor;
+        AccordRowCompactor<?, ?> compactor;
         final Version userVersion;
 
         public AccordJournalPurger(AccordCompactionInfos compactionInfos, Version version, ColumnFamilyStore cfs)
@@ -899,7 +903,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
                         compactor = new AccordCommandRowCompactor(infos, userVersion, nowInSec);
                         break;
                     case TOPOLOGY_UPDATE:
-                        compactor = new TopologyCompactor((FlyweightSerializer<Object, AccordTopologyUpdate.Accumulator>) key.type.serializer, userVersion, infos.minEpoch);
+                        compactor = new TopologyCompactor(userVersion, infos.minEpoch);
                         break;
                     default:
                         compactor = new AccordMergingCompactor(key.type.serializer, userVersion);
@@ -939,11 +943,11 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         }
     }
 
-    static abstract class AccordRowCompactor<T extends FlyweightImage>
+    static abstract class AccordRowCompactor<V, T extends Merger>
     {
-        final FlyweightSerializer<Object, T> serializer;
+        final MergeSerializer<V, ? super T, T> serializer;
 
-        AccordRowCompactor(FlyweightSerializer<Object, T> serializer)
+        AccordRowCompactor(MergeSerializer<V, ? super T, T> serializer)
         {
             this.serializer = serializer;
         }
@@ -953,15 +957,15 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         abstract UnfilteredRowIterator result(JournalKey journalKey, DecoratedKey partitionKey) throws IOException;
     }
 
-    static class TopologyCompactor extends AccordMergingCompactor<AccordTopologyUpdate.Accumulator>
+    static class TopologyCompactor extends AccordMergingCompactor<TopologyRecord, MergeSerializers.TopologyMerger>
     {
         TopologyImage lastImage;
         boolean hasWritten;
         final long minEpoch;
 
-        TopologyCompactor(FlyweightSerializer<Object, AccordTopologyUpdate.Accumulator> serializer, Version userVersion, long minEpoch)
+        TopologyCompactor(Version userVersion, long minEpoch)
         {
-            super(serializer, userVersion);
+            super(INSTANCE, userVersion);
             this.minEpoch = minEpoch;
         }
 
@@ -1004,7 +1008,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         }
     }
 
-    static class AccordMergingCompactor<T extends FlyweightImage> extends AccordRowCompactor<T>
+    static class AccordMergingCompactor<V, T extends Merger> extends AccordRowCompactor<V, T>
     {
         final T builder;
         final Version userVersion;
@@ -1012,7 +1016,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         long lastDescriptor;
         int lastOffset;
 
-        AccordMergingCompactor(FlyweightSerializer<Object, T> serializer, Version userVersion)
+        AccordMergingCompactor(MergeSerializer<V, ? super T, T> serializer, Version userVersion)
         {
             super(serializer);
             this.builder = serializer.mergerFor();
@@ -1072,7 +1076,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
     static class AccordCommandRowEntry
     {
-        final AccordJournal.Builder builder = new AccordJournal.Builder();
+        final CommandChanges builder = new CommandChanges();
         Row row;
         boolean modified;
 
@@ -1094,7 +1098,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         }
     }
 
-    static class AccordCommandRowCompactor extends AccordRowCompactor<AccordJournal.Builder>
+    static class AccordCommandRowCompactor extends AccordRowCompactor<CommandChangeWriter, CommandChanges>
     {
         static final Object[] rowTemplate = BTree.build(BulkIterator.of(new Object[2]), 2, UpdateFunction.noOp);
         final long timestamp = ClientState.getTimestamp();
@@ -1103,14 +1107,14 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         final ColumnData userVersionCell;
         final long nowInSec;
 
-        final AccordJournal.Builder mainBuilder = new AccordJournal.Builder();
+        final CommandChanges mainBuilder = new CommandChanges();
         final List<AccordCommandRowEntry> entries = new ArrayList<>();
         final ArrayDeque<AccordCommandRowEntry> reuseEntries = new ArrayDeque<>();
         AccordCompactionInfo info;
 
         AccordCommandRowCompactor(AccordCompactionInfos infos, Version userVersion, long nowInSec)
         {
-            super((FlyweightSerializer<Object, AccordJournal.Builder>) JournalKey.Type.COMMAND_DIFF.serializer);
+            super((CommandChangeSerializer) JournalKey.Type.COMMAND_DIFF.serializer);
             this.infos = infos;
             this.userVersion = userVersion;
             this.userVersionCell = BufferCell.live(AccordKeyspace.JournalColumns.user_version, timestamp, Int32Type.instance.decompose(userVersion.version));

src/java/org/apache/cassandra/db/compaction/CursorCompactor.java

@@ -157,6 +157,9 @@ public class CursorCompactor extends CompactionInfo.Holder
 
     public static boolean unsupportedMetadata(TableMetadata metadata)
     {
+        if (metadata.keyspace.equals(SchemaConstants.ACCORD_KEYSPACE_NAME))
+            return true;
+
         if (!metadata.partitioner.supportsReusableKeys())
         {
             if (LOGGER.isDebugEnabled()) logDebugReason(metadata, "Incompatible partitioner, does not support reusable keys:" + metadata.partitioner.getClass().getSimpleName());

src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java

@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
+import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -56,7 +56,7 @@ public abstract class AbstractMemtable implements Memtable
     // The smallest local deletion time for all partitions in this memtable
     protected AtomicLong minLocalDeletionTime = new AtomicLong(Long.MAX_VALUE);
     private final long id = nextId.incrementAndGet();
-    private Map<Object, Consumer<TableMetadata>> onFlush = ImmutableMap.of();
+    private Map<Object, BiConsumer<Long, TableMetadata>> onFlush = ImmutableMap.of();
     // Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not
     // correct for their usage.
 
@@ -154,7 +154,7 @@ public abstract class AbstractMemtable implements Memtable
     }
 
     @Override
-    public synchronized <T extends Consumer<TableMetadata>> T ensureFlushListener(Object key, Supplier<T> factory)
+    public synchronized <T extends BiConsumer<Long, TableMetadata>> T ensureFlushListener(Object key, Supplier<T> factory)
     {
         if (onFlush == null)
             return null;
@@ -163,7 +163,7 @@ public abstract class AbstractMemtable implements Memtable
         if (null == listener)
         {
             listener = factory.get();
-            onFlush = ImmutableMap.<Object, Consumer<TableMetadata>>builder()
+            onFlush = ImmutableMap.<Object, BiConsumer<Long, TableMetadata>>builder()
                                   .putAll(onFlush)
                                   .put(key, listener)
                                   .build();
@@ -173,13 +173,13 @@ public abstract class AbstractMemtable implements Memtable
 
     public void notifyFlushed()
     {
-        Collection<Consumer<TableMetadata>> run;
+        Collection<BiConsumer<Long, TableMetadata>> run;
         synchronized (this)
         {
             run = onFlush.values();
             onFlush = null;
         }
-        run.forEach(c -> c.accept(metadata()));
+        run.forEach(c -> c.accept(id, metadata()));
     }
 
     protected static class ColumnsCollector

src/java/org/apache/cassandra/db/memtable/Memtable.java

@@ -19,7 +19,7 @@
 package org.apache.cassandra.db.memtable;
 
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
+import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
 import javax.annotation.concurrent.NotThreadSafe;
@@ -425,7 +425,7 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource, CellSo
     }
 
     // returns null if already flushed
-    <T extends Consumer<TableMetadata>> T ensureFlushListener(Object key, Supplier<T> factory);
+    <T extends BiConsumer<Long, TableMetadata>> T ensureFlushListener(Object key, Supplier<T> factory);
     void notifyFlushed();
 
     /**

src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java

@@ -50,9 +50,9 @@ import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableTxnSingleStreamWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordTopology;
 import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
+import org.apache.cassandra.service.accord.topology.AccordTopology;
 import org.apache.cassandra.streaming.IncomingStream;
 import org.apache.cassandra.streaming.StreamReceiver;
 import org.apache.cassandra.streaming.StreamSession;

src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java

@@ -36,6 +36,7 @@ import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.IntFunction;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 
@@ -61,6 +62,7 @@ import accord.impl.CommandChange;
 import accord.impl.progresslog.DefaultProgressLog;
 import accord.impl.progresslog.DefaultProgressLog.ModeFlag;
 import accord.impl.progresslog.TxnStateKind;
+import accord.local.CatchupHard;
 import accord.local.Cleanup;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -105,6 +107,7 @@ import accord.utils.async.AsyncChains;
 import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 
+import org.apache.cassandra.config.AccordSpec.JournalSpec.ReplayMode;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -135,7 +138,6 @@ import org.apache.cassandra.service.accord.AccordCacheEntry;
 import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordCommandStores;
 import org.apache.cassandra.service.accord.AccordExecutor;
-import org.apache.cassandra.service.accord.AccordJournal;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordOperations;
 import org.apache.cassandra.service.accord.AccordService;
@@ -153,6 +155,7 @@ import org.apache.cassandra.service.accord.debug.DebugTxnDepsAll;
 import org.apache.cassandra.service.accord.debug.DebugTxnDepsOrdered;
 import org.apache.cassandra.service.accord.debug.DebugTxnGraph;
 import org.apache.cassandra.service.accord.debug.TxnKindsAndDomains;
+import org.apache.cassandra.service.accord.journal.AccordJournal;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
 import org.apache.cassandra.service.consensus.migration.TableMigrationState;
 import org.apache.cassandra.tcm.ClusterMetadata;
@@ -650,17 +653,17 @@ public class AccordDebugKeyspace extends VirtualKeyspace
         public void collect(PartitionsCollector collector)
         {
             DurableBefore durableBefore = AccordService.unsafeInstance().node().durableBefore();
-            durableBefore.foldlWithBounds(
-                (entry, ignore, start, end) -> {
-                    TableId tableId = (TableId) start.prefix();
-                    collector.row(tableId.toString(), printToken(start))
+            durableBefore.foldl(
+                (entry, ignore) -> {
+                    TableId tableId = (TableId) entry.prefix();
+                    collector.row(tableId.toString(), printToken(entry.start()))
                              .lazyCollect(columns -> {
-                                  columns.add("token_end", end, AccordDebugKeyspace::printToken)
-                                         .add("quorum", entry.quorumBefore, TO_STRING)
-                                         .add("universal", entry.universalBefore, TO_STRING);
+                                  columns.add("token_end", entry.end(), AccordDebugKeyspace::printToken)
+                                         .add("quorum", entry.quorum, TO_STRING)
+                                         .add("universal", entry.universal, TO_STRING);
                              });
                     return null;
-                }, null, ignore -> false);
+                }, null);
         }
     }
 
@@ -834,16 +837,16 @@ public class AccordDebugKeyspace extends VirtualKeyspace
                 String tableIdStr = tableId.toString();
 
                 collector.partition(commandStoreId).collect(rows -> {
-                    maxConflicts.foldlWithBounds(
-                        (timestamp, rs, start, end) -> {
-                            rows.add(printToken(start))
+                    maxConflicts.foldl(
+                        (entry, rs) -> {
+                            rows.add(printToken(entry.start()))
                                 .lazyCollect(columns -> {
-                                    columns.add("token_end", end, AccordDebugKeyspace::printToken)
+                                    columns.add("token_end", entry.end(), AccordDebugKeyspace::printToken)
                                            .add("table_id", tableIdStr)
-                                           .add("timestamp", timestamp, TO_STRING);
+                                           .add("timestamp", entry, TO_STRING);
                                 });
                              return rows;
-                        }, rows, ignore -> false
+                        }, rows
                     );
                 });
             }
@@ -1565,7 +1568,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
                 }
             }
 
-            accord.journal().forEach(key -> collect(collector, accord, key), min, max, true);
+            accord.journal().forEach(key -> collect(collector, accord, key), min, max, true, 0);
         }
 
         abstract void collect(PartitionsCollector collector, AccordService accord, JournalKey key);
@@ -2020,6 +2023,9 @@ public class AccordDebugKeyspace extends VirtualKeyspace
             SET_PROGRESS_LOG_MODE("Set the specified progress log mode."),
             UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."),
             TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."),
+            REPLAY("Run journal replay for all transactions"),
+            REBOOTSTRAP("Rebootstrap the command store. This invalidates the local journal, synchronises its data via data repair and rejoins the distributed state machine."),
+            HARD_CATCHUP("Hard catchup the command store. This invalidates the local journal for any ranges not up to date with some quorum, synchronises its data via data repair and rejoins the distributed state machine."),
             ;
 
             final String description;
@@ -2071,12 +2077,16 @@ public class AccordDebugKeyspace extends VirtualKeyspace
             if (op == null)
                 throw new IllegalArgumentException("Must specify 'op'");
 
+            final AccordService accord = (AccordService) AccordService.unsafeInstance();
+            final Node node = accord.node();
             final Function<CommandStore, AsyncResult<?>> function;
+            Supplier<AsyncResult<?>> allFunction = null;
             switch (op)
             {
                 default: throw new UnhandledEnum(op);
                 case SET_PROGRESS_LOG_MODE:
                 case UNSET_PROGRESS_LOG_MODE:
+                {
                     if (param == null)
                         throw new IllegalArgumentException("Must specify 'param' for " + op);
                     ModeFlag mode = tryParse(param, true, ModeFlag.class, ModeFlag::valueOf);
@@ -2088,25 +2098,53 @@ public class AccordDebugKeyspace extends VirtualKeyspace
                         return AsyncResults.success(null);
                     };
                     break;
+                }
                 case TRY_EXECUTE_LISTENING:
-                    if (param != null)
-                        throw new IllegalArgumentException("'param' is not supported for " + op);
-                    function = CommandStore::operatorTryToExecuteListeningTxns;
+                {
+                    boolean loop;
+                    if (param == null) loop = false;
+                    else if (param.equalsIgnoreCase("loop")) loop = true;
+                    else throw new InvalidRequestException("Unknown param for " + CommandStoreOp.TRY_EXECUTE_LISTENING + ": '" + param + "'; expect only 'loop' or missing");
+
+                    function = commandStore -> commandStore.tryToExecuteListeningTxns(loop);
+                    break;
+                }
+                case REPLAY:
+                {
+                    ReplayMode replayMode = tryParse(param, true, ReplayMode.class, ReplayMode::valueOf);
+                    function = commandStore -> {
+                        accord.journal().replay(commandStore, replayMode, 0L);
+                        return AsyncResults.success(null);
+                    };
+                    break;
+                }
+                case REBOOTSTRAP:
+                    allFunction = () -> node.commandStores().rebootstrap(node);
+                    function = commandStore -> commandStore.rebootstrap(node);
+                    break;
+                case HARD_CATCHUP:
+                    allFunction = () -> CatchupHard.catchup(node, Arrays.asList(node.commandStores().all())).beginAsResult();
+                    function = commandStore -> CatchupHard.catchup(node, Collections.singletonList(commandStore)).beginAsResult();
                     break;
             }
 
             AsyncResult<?> result;
             if (commandStoreId < 0)
             {
-                List<AsyncResult<?>> results = new ArrayList<>();
-                AccordService.unsafeInstance().node()
-                             .commandStores()
-                             .forAllUnsafe(commandStore -> results.add(function.apply(commandStore)));
-                result = AsyncResults.allOf(results);
+                if (allFunction == null)
+                {
+                    allFunction = () -> {
+                        List<AsyncResult<?>> results = new ArrayList<>();
+                        for (CommandStore commandStore : node.commandStores().all())
+                            results.add(function.apply(commandStore));
+                        return AsyncResults.allOf(results);
+                    };
+                }
+                result = allFunction.get();
             }
             else
             {
-                result = function.apply(AccordService.unsafeInstance().node().commandStores().forId(commandStoreId));
+                result = function.apply(node.commandStores().forId(commandStoreId));
             }
 
             AccordService.getBlocking(result);

src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java

@@ -57,10 +57,10 @@ import org.apache.cassandra.io.util.ChecksumedSequentialWriter;
 import org.apache.cassandra.io.util.FileHandle;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.serializers.UUIDSerializer;
-import org.apache.cassandra.service.accord.AccordJournal;
-import org.apache.cassandra.service.accord.AccordJournalTable;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.journal.AccordJournal;
+import org.apache.cassandra.service.accord.journal.CommandChanges;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.service.accord.serializers.Version;
@@ -160,7 +160,7 @@ public class RouteIndexFormat
             return null;
         ByteBuffer record = recordCell.buffer();
         Version user_version = Version.fromVersion(Int32Type.instance.compose(user_versionCell.buffer()));
-        AccordJournal.Builder builder = extract(txnId, record, user_version);
+        CommandChanges builder = extract(txnId, record, user_version);
         StoreParticipants participants = builder.participants();
         if (participants == null)
             return null;
@@ -179,10 +179,10 @@ public class RouteIndexFormat
         }
     }
 
-    public static AccordJournal.Builder extract(TxnId txnId, ByteBuffer record, Version userVersion)
+    public static CommandChanges extract(TxnId txnId, ByteBuffer record, Version userVersion)
     {
-        AccordJournal.Builder builder = new AccordJournal.Builder(txnId, AccordJournal.Load.ALL);
-        AccordJournalTable.readBuffer(record, builder::deserializeNext, userVersion);
+        CommandChanges builder = new CommandChanges(txnId, AccordJournal.Load.ALL);
+        builder.deserializeNext(record, userVersion);
         return builder;
     }
 

src/java/org/apache/cassandra/index/accord/RouteJournalIndex.java

@@ -89,10 +89,10 @@ import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.accord.AccordJournalTable;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.api.TokenKey;
+import org.apache.cassandra.service.accord.journal.RangeSearchManager;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -394,7 +394,7 @@ public class RouteJournalIndex implements Index, INotificationConsumer
         Timestamp maxTxnId = TxnId.MAX;
         for (RowFilter.Expression e : expressions)
         {
-            if (e.column() == AccordJournalTable.SyntheticColumn.participants.metadata)
+            if (e.column() == RangeSearchManager.SyntheticColumn.participants.metadata)
             {
                 switch (e.operator())
                 {
@@ -410,11 +410,11 @@ public class RouteJournalIndex implements Index, INotificationConsumer
                         return null;
                 }
             }
-            else if (e.column() == AccordJournalTable.SyntheticColumn.store_id.metadata && e.operator() == Operator.EQ)
+            else if (e.column() == RangeSearchManager.SyntheticColumn.store_id.metadata && e.operator() == Operator.EQ)
             {
                 storeId = Int32Type.instance.compose(e.getIndexValue());
             }
-            else if (e.column() == AccordJournalTable.SyntheticColumn.txn_id.metadata)
+            else if (e.column() == RangeSearchManager.SyntheticColumn.txn_id.metadata)
             {
                 switch (e.operator())
                 {

src/java/org/apache/cassandra/index/sai/plan/QueryViewBuilder.java

@@ -92,6 +92,7 @@ public class QueryViewBuilder
         public void close()
         {
             referencedIndexes.forEach(SSTableIndex::releaseQuietly);
+            referencedIndexes.clear();
         }
     }
 

src/java/org/apache/cassandra/io/util/CompressedFrameDataInputPlus.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.util.List;
+import java.util.zip.Checksum;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.UnversionedSerializer;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+import org.apache.cassandra.utils.CollectionSerializers;
+import org.apache.cassandra.utils.Crc;
+
+import static org.apache.cassandra.io.compress.ZstdCompressor.DEFAULT_COMPRESSION_LEVEL;
+import static org.apache.cassandra.io.util.CompressedFrameDataOutputPlus.DEFAULT_FRAME_SIZE;
+import static org.apache.cassandra.io.util.CompressedFrameDataOutputPlus.SIZE_OF_HEADER;
+
+public class CompressedFrameDataInputPlus extends RebufferingInputStream
+{
+    final ICompressor compressor;
+    final Checksum checksum;
+    final ReadableByteChannel channel;
+    ByteBuffer compressed;
+
+    protected CompressedFrameDataInputPlus(int frameSize, ReadableByteChannel channel, ICompressor compressor, Checksum checksum)
+    {
+        super(compressor.preferredBufferType().allocate(frameSize));
+        this.compressor = compressor;
+        this.checksum = checksum;
+        this.channel = channel;
+        this.compressed = compressor.preferredBufferType().allocate(frameSize);
+        buffer.limit(0);
+    }
+
+    @Override
+    protected void reBuffer() throws IOException
+    {
+        compressed.position(0);
+        compressed.limit(SIZE_OF_HEADER);
+        while (channel.read(compressed) >= 0 && compressed.hasRemaining());
+        compressed.flip();
+        long headerChecksum = compressed.getLong();
+        int length = compressed.getShort();
+
+        boolean decompress = length >= 0;
+        if (!decompress)
+            length = -1 - length;
+
+        compressed.clear();
+        compressed.limit(length);
+        while (compressed.hasRemaining())
+        {
+            if (channel.read(compressed) < 0)
+                throw new EOFException("Incomplete file: header stipulated " + length + " bytes but found only " + compressed.position());
+        }
+        compressed.flip();
+        this.checksum.update(compressed);
+        compressed.flip();
+        long dataChecksum = checksum.getValue();
+        if (headerChecksum != dataChecksum)
+            throw new IOException("Invalid checksum: " + headerChecksum + " != " + dataChecksum);
+
+        buffer.clear();
+        if (decompress) compressor.uncompress(compressed, buffer);
+        else buffer.put(compressed);
+        buffer.flip();
+    }
+
+    public static <T> T read(File file, IVersionedSerializer<T> serializer) throws IOException
+    {
+        try (CompressedFrameDataInputPlus in = new CompressedFrameDataInputPlus(DEFAULT_FRAME_SIZE, file.newReadChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            int version = in.readUnsignedVInt32();
+            return serializer.deserialize(in, version);
+        }
+    }
+
+    public static <T> T readOne(File file, UnversionedSerializer<T> serializer) throws IOException
+    {
+        try (CompressedFrameDataInputPlus in = new CompressedFrameDataInputPlus(DEFAULT_FRAME_SIZE, file.newReadChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            int version = in.readUnsignedVInt32();
+            if (version != 0)
+                throw new IOException("Expected version 0 for unversioned serializer");
+            return serializer.deserialize(in);
+        }
+    }
+
+    public static <T> List<T> readList(File file, UnversionedSerializer<T> serializer) throws IOException
+    {
+        try (CompressedFrameDataInputPlus in = new CompressedFrameDataInputPlus(DEFAULT_FRAME_SIZE, file.newReadChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            int version = in.readUnsignedVInt32();
+            if (version != 0)
+                throw new IOException("Expected version 0 for unversioned serializer");
+            return CollectionSerializers.deserializeList(in, serializer);
+        }
+    }
+}

src/java/org/apache/cassandra/io/util/CompressedFrameDataOutputPlus.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+import java.util.List;
+import java.util.Objects;
+import java.util.zip.Checksum;
+
+import com.google.common.primitives.Shorts;
+
+import accord.utils.Invariants;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.UnversionedSerializer;
+import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+import org.apache.cassandra.utils.CollectionSerializers;
+import org.apache.cassandra.utils.Crc;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+
+import static org.apache.cassandra.io.compress.ZstdCompressor.DEFAULT_COMPRESSION_LEVEL;
+
+public class CompressedFrameDataOutputPlus extends BufferedDataOutputStreamPlus
+{
+    static final int SIZE_OF_HEADER = 10;
+    static final int DEFAULT_FRAME_SIZE = 16 << 10;
+
+    private final ICompressor compressor;
+    private final Checksum checksum;
+    private ByteBuffer compressed;
+    protected CompressedFrameDataOutputPlus(int frameSize, WritableByteChannel out, ICompressor compressor, Checksum checksum)
+    {
+        super(out, compressor.preferredBufferType().allocate(frameSize));
+        this.compressor = compressor;
+        this.compressed = compressor.preferredBufferType().allocate(frameSize + SIZE_OF_HEADER);
+        this.checksum = checksum;
+        if (frameSize > Short.MAX_VALUE)
+            throw new IllegalArgumentException("Frame size too large");
+    }
+
+    @Override
+    protected void doFlush(int count) throws IOException
+    {
+        buffer.flip();
+        compressed.clear();
+        compressed.position(SIZE_OF_HEADER);
+        compressor.compress(buffer, compressed);
+        compressed.flip();
+        int limit = compressed.limit();
+        int length = limit - SIZE_OF_HEADER;
+        if (length > buffer.limit())
+        {
+            length = -(1 + buffer.limit());
+            compressed.clear();
+            compressed.position(SIZE_OF_HEADER);
+            buffer.position(0);
+            compressed.put(buffer);
+            compressed.flip();
+        }
+        compressed.putShort(SIZE_OF_HEADER - 2, Shorts.checkedCast(length));
+        compressed.position(SIZE_OF_HEADER);
+        checksum.update(compressed);
+        compressed.putLong(0, checksum.getValue());
+        compressed.position(0);
+        while (compressed.hasRemaining())
+            channel.write(compressed);
+        buffer.clear();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        super.close();
+        MemoryUtil.clean(compressed);
+        compressed = null;
+    }
+
+    public static <T> void write(File file, T value, IVersionedSerializer<T> serializer, int version) throws IOException
+    {
+        try (CompressedFrameDataOutputPlus out = new CompressedFrameDataOutputPlus(DEFAULT_FRAME_SIZE, file.newReadWriteChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            out.writeUnsignedVInt32(version);
+            serializer.serialize(value, out, version);
+        }
+    }
+
+    public static <T> void writeOne(File file, T value, UnversionedSerializer<T> serializer) throws IOException
+    {
+        try (CompressedFrameDataOutputPlus out = new CompressedFrameDataOutputPlus(DEFAULT_FRAME_SIZE, file.newReadWriteChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            out.writeUnsignedVInt32(0);
+            serializer.serialize(value, out);
+        }
+        if (Invariants.isParanoid())
+        {
+            try
+            {
+                T reconcile = CompressedFrameDataInputPlus.readOne(file, serializer);
+                Invariants.require(Objects.equals(value, reconcile));
+            }
+            catch (IOException e) {}
+        }
+    }
+
+    public static <T> void writeList(File file, List<T> value, UnversionedSerializer<T> serializer) throws IOException
+    {
+        try (CompressedFrameDataOutputPlus out = new CompressedFrameDataOutputPlus(DEFAULT_FRAME_SIZE, file.newReadWriteChannel(), ZstdCompressor.getOrCreate(DEFAULT_COMPRESSION_LEVEL), Crc.crc32()))
+        {
+            out.writeUnsignedVInt32(0);
+            CollectionSerializers.serializeList(value, out, serializer);
+        }
+        if (Invariants.isParanoid())
+        {
+            try
+            {
+                List<T> reconcile = CompressedFrameDataInputPlus.readList(file, serializer);
+                Invariants.require(Objects.equals(value, reconcile));
+            }
+            catch (IOException e) {}
+        }
+    }
+}

src/java/org/apache/cassandra/journal/Compactor.java

@@ -116,13 +116,17 @@ public final class Compactor<K, V> implements Runnable, Shutdownable
     public void shutdown()
     {
         logger.debug("Shutting down " + executor);
+        if (scheduled != null)
+            scheduled.cancel(false);
         executor.shutdown();
     }
 
     @Override
     public Object shutdownNow()
     {
-        return executor.shutdownNow();
+        // if we call executor.shutdownNow() we can cause ClosedByInterruptException
+        shutdown();
+        return null;
     }
 
     @Override

src/java/org/apache/cassandra/journal/DumpUtil.java

@@ -20,6 +20,8 @@ package org.apache.cassandra.journal;
 
 import java.util.function.Consumer;
 
+import static org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy.FAIL;
+
 /**
  * Helper file to avoid exposing components outside their package-local visibility scope
  */
@@ -37,6 +39,6 @@ public class DumpUtil
 
     public static <K, V> StaticSegment<K, V> open(Descriptor descriptor, KeySupport<K> keySupport)
     {
-        return StaticSegment.open(descriptor, keySupport);
+        return StaticSegment.open(descriptor, keySupport, FAIL);
     }
 }

src/java/org/apache/cassandra/journal/EntrySerializer.java

@@ -28,8 +28,6 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Crc;
 
-import static org.apache.cassandra.journal.Journal.validateCRC;
-
 /**
  * Entry format:
  *
@@ -73,10 +71,10 @@ public final class EntrySerializer
         out.limit(headerCrcPosition);
         crc.update(out);
         out.limit(recordEnd);
-        out.putInt((int) crc.getValue());
+        out.putInt(nonZeroCrc(crc));
         crc.update(out);
         out.limit(recordEnd + 4);
-        out.putInt((int) crc.getValue());
+        out.putInt(nonZeroCrc(crc));
     }
 
     // we reuse record as the value we return
@@ -173,11 +171,15 @@ public final class EntrySerializer
     public static class MaybeRecoverableJournalError extends IOException
     {
         public final int knownLength;
+        public final int readCrc;
+        public final int computedCrc;
 
-        public MaybeRecoverableJournalError(int knownLength, Throwable cause)
+        public MaybeRecoverableJournalError(int knownLength, Crc.InvalidCrc cause)
         {
             super(cause);
             this.knownLength = knownLength;
+            this.readCrc = cause.read;
+            this.computedCrc = cause.computed;
         }
 
         @Override
@@ -244,4 +246,16 @@ public final class EntrySerializer
             value = null;
         }
     }
+
+    static int nonZeroCrc(CRC32 crc)
+    {
+        int v = (int) crc.getValue();
+        return v == 0 ? 1 : v;
+    }
+
+    static void validateCRC(CRC32 crc, int readCRC) throws Crc.InvalidCrc
+    {
+        if (readCRC != nonZeroCrc(crc))
+            throw new Crc.InvalidCrc(readCRC, nonZeroCrc(crc));
+    }
 }

src/java/org/apache/cassandra/journal/Flusher.java

@@ -17,6 +17,9 @@
  */
 package org.apache.cassandra.journal;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
@@ -102,19 +105,24 @@ final class Flusher<K, V>
         flushExecutor = executorFactory().infiniteLoop(flushExecutorName, new FlushRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
     }
 
-    void shutdown() throws InterruptedException
+    void shutdown()
     {
-        logger.debug("Shutting down " + flushExecutor + " and awaiting termination");
+        logger.debug("Shutting down " + flushExecutor);
         flushExecutor.shutdown();
-        flushExecutor.awaitTermination(1, MINUTES);
         if (fsyncExecutor != null)
         {
-            logger.debug("Shutting down " + fsyncExecutor + " and awaiting termination");
-            fsyncExecutor.shutdownNow(); // `now` to interrupt potentially parked runnable
-            fsyncExecutor.awaitTermination(1, MINUTES);
+            logger.debug("Shutting down " + fsyncExecutor);
+            fsyncExecutor.shutdown(); // `now` to interrupt potentially parked runnable
         }
     }
 
+    List<Interruptible> executors()
+    {
+        if (fsyncExecutor != null)
+            return Arrays.asList(flushExecutor, fsyncExecutor);
+        return Collections.singletonList(flushExecutor);
+    }
+
     @Simulate(with={MONITORS,GLOBAL_CLOCK,LOCK_SUPPORT})
     // waits for writes to complete before triggering an fsync
     private class FlushRunnable implements Interruptible.Task
@@ -140,6 +148,10 @@ final class Flusher<K, V>
                 {
                     doRun(state);
                 }
+                catch (InterruptedException t)
+                {
+                    throw t;
+                }
                 catch (Throwable t)
                 {
                     if (!journal.handleError("Failed to flush segments to disk", t))
@@ -189,8 +201,8 @@ final class Flusher<K, V>
 
             public void doRun(Interruptible.State state) throws InterruptedException
             {
-                if (state == NORMAL) awaitWork();
-                else if (!hasWork()) return;
+                if (state == NORMAL)
+                    awaitWork();
 
                 if (fsyncing == null)
                     fsyncing = journal.oldestActiveSegment();
@@ -260,6 +272,10 @@ final class Flusher<K, V>
             {
                 doRun(state);
             }
+            catch (InterruptedException t)
+            {
+                throw t;
+            }
             catch (Throwable t)
             {
                 if (!journal.handleError("Failed to flush segments to disk", t))
@@ -534,8 +550,8 @@ final class Flusher<K, V>
             {
                 signal.awaitThrowUncheckedOnInterrupt();
 
-                Journal.State state = journal.state.get();
-                Invariants.require(state == Journal.State.NORMAL,
+                Journal.State state = journal.getState();
+                Invariants.require(state.compareTo(Journal.State.STOPPED_READABLE) < 0,
                                       "Thread %s outlived journal, which is in %s state", Thread.currentThread(), state);
             }
             else
@@ -544,6 +560,24 @@ final class Flusher<K, V>
         while (fsyncFinishedFor < flushTime);
     }
 
+    void awaitFsync(ActiveSegment<K, V> segment, int fsyncedTo)
+    {
+        while (true)
+        {
+            Journal.State state = journal.getState();
+            Invariants.require(state.compareTo(Journal.State.STOPPED_READABLE) < 0,
+                               "Thread %s outlived journal, which is in %s state", Thread.currentThread(), state);
+
+            WaitQueue.Signal signal = fsyncComplete.register();
+            if (segment.fsyncedTo() >= fsyncedTo)
+            {
+                signal.cancel();
+                break;
+            }
+            signal.awaitThrowUncheckedOnInterrupt();
+        }
+    }
+
     private long flushPeriodNanos()
     {
         return params.flushPeriod(NANOSECONDS);

src/java/org/apache/cassandra/journal/InMemoryIndex.java

@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
 
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy;
 import org.apache.cassandra.journal.StaticSegment.SequentialReader;
 
 /**
@@ -154,11 +155,11 @@ public final class InMemoryIndex<K> extends Index<K>
         tmpFile.move(descriptor.fileFor(Component.INDEX));
     }
 
-    static <K> InMemoryIndex<K> rebuild(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> InMemoryIndex<K> rebuild(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
         InMemoryIndex<K> index = new InMemoryIndex<>(keySupport, new TreeMap<>(keySupport));
 
-        try (SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, fsyncedLimit))
+        try (SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, fsyncedLimit, crcFailurePolicy))
         {
             while (reader.advance())
                 index.update(reader.key(), reader.offset, reader.buffer.position() - reader.offset);

src/java/org/apache/cassandra/journal/Journal.java

@@ -27,10 +27,11 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import java.util.concurrent.locks.LockSupport;
-import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -48,7 +49,6 @@ import accord.utils.Invariants;
 import org.apache.cassandra.concurrent.Interruptible;
 import org.apache.cassandra.concurrent.Interruptible.TerminateException;
 import org.apache.cassandra.concurrent.SequentialExecutorPlus;
-import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -59,6 +59,7 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.CloseableIterator;
 import org.apache.cassandra.utils.Crc;
+import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.MergeIterator;
 import org.apache.cassandra.utils.Simulate;
@@ -71,7 +72,7 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NO
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.SYNCHRONIZED;
 import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
 import static org.apache.cassandra.concurrent.Interruptible.State.NORMAL;
-import static org.apache.cassandra.concurrent.Interruptible.State.SHUTTING_DOWN;
+import static org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy.FAIL;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.Simulate.With.MONITORS;
 import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
@@ -89,7 +90,7 @@ import static org.apache.cassandra.utils.concurrent.WaitQueue.newWaitQueue;
               must be fixed-size and byte-order comparable
  */
 @Simulate(with=MONITORS)
-public class Journal<K, V> implements Shutdownable
+public class Journal<K, V>
 {
     private static final Logger logger = LoggerFactory.getLogger(Journal.class);
 
@@ -104,10 +105,10 @@ public class Journal<K, V> implements Shutdownable
 
     final Flusher<K, V> flusher;
     final Compactor<K, V> compactor;
+    final AllocateRunnable allocateRunnable = new AllocateRunnable();
     Interruptible allocator;
     SequentialExecutorPlus closer, releaser;
 
-    volatile long replayLimit;
     final AtomicLong nextSegmentId = new AtomicLong();
 
     private volatile ActiveSegment<K, V> currentSegment = null;
@@ -117,12 +118,11 @@ public class Journal<K, V> implements Shutdownable
 
     private final AtomicReference<Segments<K, V>> segments = new AtomicReference<>();
 
-    final AtomicReference<State> state = new AtomicReference<>(State.UNINITIALIZED);
+    private volatile State state = State.UNINITIALIZED;
+    private static final AtomicReferenceFieldUpdater<Journal, State> stateUpdater = AtomicReferenceFieldUpdater.newUpdater(Journal.class, State.class, "state");
 
-    // TODO (expected): we do not need wait queues here, we can just wait on a signal on a segment while its byte buffer is being allocated
     private final WaitQueue segmentPrepared = newWaitQueue();
-    private final WaitQueue allocatorThreadWaitQueue = newWaitQueue();
-    private final BooleanSupplier allocatorThreadWaitCondition = () -> (availableSegment == null);
+    private volatile Thread waitingAllocatorThread;
 
     private final FlusherCallbacks flusherCallbacks;
 
@@ -199,37 +199,53 @@ public class Journal<K, V> implements Shutdownable
         this.compactor = new Compactor<>(this, segmentCompactor);
     }
 
+    public long peekSegmentId()
+    {
+        return nextSegmentId.get();
+    }
+
     public void onDurable(RecordPointer recordPointer, Runnable runnable)
     {
         flusherCallbacks.submit(recordPointer, runnable);
     }
 
-    public void start()
+    public void open()
     {
-        Invariants.require(state.compareAndSet(State.UNINITIALIZED, State.INITIALIZING),
-                              "Unexpected journal state during initialization", state);
-        metrics.register(flusher);
+        Invariants.require(stateUpdater.compareAndSet(this, State.UNINITIALIZED, State.OPENING),
+                           "Unexpected journal state before opening", state);
 
         deleteTmpFiles();
-
         List<Descriptor> descriptors = Descriptor.list(directory);
-        // find the largest existing timestamp
-        descriptors.sort(null);
-        long maxTimestamp = descriptors.isEmpty()
-                          ? Long.MIN_VALUE
-                          : descriptors.get(descriptors.size() - 1).timestamp;
-        nextSegmentId.set(replayLimit = Math.max(currentTimeMillis(), maxTimestamp + 1));
+        segments.set(Segments.of(StaticSegment.open(descriptors, keySupport, params.crcFailureOnRebuildPolicy())));
+
+        Invariants.require(stateUpdater.compareAndSet(this, State.OPENING, State.OPEN_READABLE),
+                           "Unexpected journal state once opened", state);
+    }
+
+    public void start(long maxTableDescriptor)
+    {
+        if (state == State.UNINITIALIZED)
+            open();
+
+        Invariants.require(stateUpdater.compareAndSet(this, State.OPEN_READABLE, State.STARTING),
+                              "Unexpected journal state before starting", state);
+
+        nextSegmentId.set(Math.max(currentTimeMillis(), Math.max(maxDescriptor(), maxTableDescriptor) + 1));
 
-        segments.set(Segments.of(StaticSegment.open(descriptors, keySupport)));
         closer = executorFactory().sequential(name + "-closer");
         releaser = executorFactory().sequential(name + "-releaser");
-        allocator = executorFactory().infiniteLoop(name + "-allocator", new AllocateRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
+        allocator = executorFactory().infiniteLoop(name + "-allocator", allocateRunnable, SAFE, NON_DAEMON, SYNCHRONIZED);
+
+        // we use these metrics when advancing segments, so must register first
+        metrics.register(flusher);
         advanceSegment(null);
-        Invariants.require(state.compareAndSet(State.INITIALIZING, State.NORMAL),
-                              "Unexpected journal state after initialization", state);
+
         flusher.start();
         compactor.start();
 
+        Invariants.require(stateUpdater.compareAndSet(this, State.STARTING, State.WRITEABLE),
+                           "Unexpected journal state once started", state);
+
         final int maxSegments = 100;
         if (segments.get().count(Segment::isStatic) > maxSegments)
         {
@@ -252,6 +268,29 @@ public class Journal<K, V> implements Shutdownable
         }
     }
 
+    public long maxDescriptor()
+    {
+        List<Segment<K, V>> existingSegments = segments.get().allSorted(false);
+        return existingSegments.isEmpty() ? 0 : existingSegments.get(0).descriptor.timestamp;
+    }
+
+    public State getState()
+    {
+        return state;
+    }
+
+    public boolean isReadable()
+    {
+        State state = this.state;
+        return state.compareTo(State.OPEN_READABLE) >= 0 && state.compareTo(State.STOPPED_READABLE) <= 0;
+    }
+
+    private boolean isNotStopped()
+    {
+        State state = this.state;
+        return state.compareTo(State.STARTING) >= 0 && state.compareTo(State.STOPPING) <= 0;
+    }
+
     @VisibleForTesting
     public void runCompactorForTesting()
     {
@@ -272,58 +311,72 @@ public class Journal<K, V> implements Shutdownable
             tmpFile.delete();
     }
 
-    @Override
+    public boolean hasBeenOpened()
+    {
+        return state.compareTo(State.OPEN_READABLE) >= 0;
+    }
+
     public boolean isTerminated()
     {
-        return state.get() == State.TERMINATED;
+        return state == State.STOPPED_READABLE;
     }
 
-    public void shutdown()
+    public void fsync()
     {
-        try
+        ActiveSegment<K, V> active = currentSegment;
+        int position = active.writtenToAtLeast();
+        flusher.requestExtraFlush();
+        flusher.awaitFsync(active, position);
+    }
+
+    // return the last segment that was written to
+    public Descriptor stop()
+    {
+        logger.info("Stopping journal");
+        logger.debug("Shutting down " + allocator);
+        boolean stop;
+        synchronized (allocateRunnable)
         {
-            Invariants.require(state.compareAndSet(State.NORMAL, State.SHUTDOWN),
-                                  "Unexpected journal state while trying to shut down", state);
-            logger.debug("Shutting down " + allocator + " and awaiting termination");
-            allocator.shutdown();
-            wakeAllocator(); // Wake allocator to force it into shutdown
-            // TODO (expected): why are we awaitingTermination here when we have a separate method for it?
-            allocator.awaitTermination(1, TimeUnit.MINUTES);
-            segmentPrepared.signalAll(); // Wake up all threads waiting on the new segment
-            compactor.shutdown();
-            compactor.awaitTermination(1, TimeUnit.MINUTES);
-            flusher.shutdown();
-            closeAllSegments();
-            logger.debug("Shutting down " + releaser + " and " + closer + " and awaiting termination");
-            releaser.shutdown();
-            closer.shutdown();
-            closer.awaitTermination(1, TimeUnit.MINUTES);
-            releaser.awaitTermination(1, TimeUnit.MINUTES);
-            metrics.deregister();
-            Invariants.require(state.compareAndSet(State.SHUTDOWN, State.TERMINATED),
-                                  "Unexpected journal state while trying to shut down", state);
-        }
-        catch (InterruptedException e)
-        {
-            logger.error("Could not shutdown journal", e);
+            // we synchronize on allocateRunnable to ensure it witnesses this change before the next attempt to allocate a segment
+            stop = stateUpdater.compareAndSet(this, State.WRITEABLE, State.STOPPING);
         }
+        Invariants.require(stop, "Unexpected journal state before stopping", state);
+
+        // ensure prompt shutdown, though the above state change suffices semantically
+        allocator.shutdown();
+        wakeAllocator();
+        discardAvailableSegment();
+        segmentPrepared.signalAll(); // Wake up all threads waiting on the new segment
+
+        compactor.shutdown();
+
+        currentSegment.discardUnusedTail();
+        flusher.requestExtraFlush();
+
+        Descriptor lastSegment = finaliseSegments(); // this flushes any pending writes
+
+        flusher.shutdown();
+        logger.debug("Shutting down " + releaser + " and " + closer);
+        releaser.shutdown();
+        closer.shutdown();
+        metrics.deregister();
+        Invariants.require(stateUpdater.compareAndSet(this, State.STOPPING, State.STOPPED_READABLE),
+                           "Unexpected journal state after stopping", state);
+        return lastSegment;
     }
 
-    @Override
-    public Object shutdownNow()
+    public void close()
     {
-        shutdown();
-        return null;
+        logger.info("Closing journal");
+        stateUpdater.compareAndSet(this, State.STOPPED_READABLE, State.CLOSING);
+        closeAllSegments();
+        stateUpdater.compareAndSet(this, State.CLOSING, State.CLOSED);
     }
 
-    @Override
-    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    public void awaitTerminationUntil(long deadlineNanos) throws InterruptedException, TimeoutException
     {
-        boolean r = true;
-        r &= allocator.awaitTermination(timeout, units);
-        r &= closer.awaitTermination(timeout, units);
-        r &= releaser.awaitTermination(timeout, units);
-        return r;
+        ExecutorUtils.awaitTerminationUntil(deadlineNanos, Arrays.asList(allocator, compactor, closer, releaser));
+        ExecutorUtils.awaitTerminationUntil(deadlineNanos, flusher.executors());
     }
 
     /**
@@ -470,9 +523,6 @@ public class Journal<K, V> implements Shutdownable
     {
         for (Segment<K, V> segment : segments.allSorted(false))
         {
-            if (!segment.index().mayContainId(id))
-                continue;
-
             if (segment.readLast(id, consumer))
                 return true;
         }
@@ -542,7 +592,6 @@ public class Journal<K, V> implements Shutdownable
 
     private ActiveSegment<K, V>.Allocation allocate(int entrySize)
     {
-
         ActiveSegment<K, V> segment = currentSegment;
         ActiveSegment<K, V>.Allocation alloc;
         while (null == (alloc = segment.allocate(entrySize)))
@@ -598,12 +647,12 @@ public class Journal<K, V> implements Shutdownable
             WaitQueue.Signal prepared = segmentPrepared.register(metrics.waitingOnSegmentAllocation.time(), Context::stop);
             if (availableSegment == null && currentSegment == currentActiveSegment)
             {
-                prepared.awaitThrowUncheckedOnInterrupt();
-
                 // In case we woke up due to shutdown signal or interrupt, check mode
-                State state = this.state.get();
-                if (state.ordinal() > State.NORMAL.ordinal())
+                State state = this.state;
+                if (state.ordinal() > State.WRITEABLE.ordinal())
                     throw new IllegalStateException("Can not obtain allocated segment due to shutdown " + state);
+
+                prepared.awaitThrowUncheckedOnInterrupt();
             }
             else
                 prepared.cancel();
@@ -613,7 +662,9 @@ public class Journal<K, V> implements Shutdownable
 
     private void wakeAllocator()
     {
-        allocatorThreadWaitQueue.signalAll();
+        Thread wake = waitingAllocatorThread;
+        if (wake != null)
+            LockSupport.unpark(wake);
     }
 
     private void discardAvailableSegment()
@@ -635,13 +686,10 @@ public class Journal<K, V> implements Shutdownable
         {
             if (state == NORMAL)
                 runNormal();
-            else if (state == SHUTTING_DOWN)
-                shutDown();
         }
 
         private void runNormal() throws InterruptedException
         {
-            boolean interrupted = false;
             try
             {
                 if (availableSegment != null)
@@ -649,14 +697,19 @@ public class Journal<K, V> implements Shutdownable
 
                 // synchronized to prevent thread interrupts while performing IO operations and also
                 // clear interrupted status to prevent ClosedByInterruptException in createSegment()
+                boolean interrupted;
                 synchronized (this)
                 {
+                    if (state.compareTo(State.STOPPING) >= 0)
+                        throw new TerminateException();
+
                     interrupted = Thread.interrupted();
                     availableSegment = createSegment();
-
-                    segmentPrepared.signalAll();
-                    Thread.yield();
                 }
+
+                segmentPrepared.signalAll();
+                if (interrupted) throw new InterruptedException();
+                else Thread.yield();
             }
             catch (JournalWriteError e)
             {
@@ -673,41 +726,18 @@ public class Journal<K, V> implements Shutdownable
                 TimeUnit.SECONDS.sleep(1L); // sleep for a second to avoid log spam
             }
 
-            interrupted = interrupted || Thread.interrupted();
-            if (!interrupted)
+            // If we offered a segment, wait for it to be taken before reentering the loop.
+            // There could be a new segment in next not offered, but only on failure to discard it while
+            // shutting down-- nothing more can or needs to be done in that case.
+            if (availableSegment != null)
             {
-                try
-                {
-                    // If we offered a segment, wait for it to be taken before reentering the loop.
-                    // There could be a new segment in next not offered, but only on failure to discard it while
-                    // shutting down-- nothing more can or needs to be done in that case.
-                    WaitQueue.waitOnCondition(allocatorThreadWaitCondition, allocatorThreadWaitQueue);
-                }
-                catch (InterruptedException e)
-                {
-                    interrupted = true;
-                }
-            }
-
-            if (interrupted)
-            {
-                discardAvailableSegment();
-                throw new InterruptedException();
-            }
-        }
-
-        private void shutDown() throws InterruptedException
-        {
-            try
-            {
-                // if shutdown() started and finished during segment creation, we'll be left with a
-                // segment that no one will consume; discard it
-                discardAvailableSegment();
-            }
-            catch (Throwable t)
-            {
-                handleError("Failed shutting down segment allocator", t);
-                throw new TerminateException();
+                waitingAllocatorThread = Thread.currentThread();
+                boolean interrupted = false;
+                while (availableSegment != null && !(interrupted = Thread.interrupted()))
+                    LockSupport.park();
+                waitingAllocatorThread = null;
+                if (interrupted)
+                    throw new InterruptedException();
             }
         }
     }
@@ -722,7 +752,8 @@ public class Journal<K, V> implements Shutdownable
     {
         Segments<K, V> segments = swapSegments(ignore -> Segments.none());
 
-        for (Segment<K, V> segment : segments.all())
+        List<Segment<K, V>> all = segments.allSorted(false);
+        for (Segment<K, V> segment : all)
         {
             if (segment.isActive())
                 ((ActiveSegment<K, V>) segment).closeAndIfEmptyDiscard(this);
@@ -731,6 +762,24 @@ public class Journal<K, V> implements Shutdownable
         }
     }
 
+    private Descriptor finaliseSegments()
+    {
+        while (true)
+        {
+            ActiveSegment<K, V> oldestActive = oldestActiveSegment();
+            oldestActive.discardUnusedTail();
+            flusher.awaitFsync(oldestActive, oldestActive.writtenToAtLeast());
+            if (oldestActive == currentSegment)
+                break;
+        }
+
+        currentSegment.persistComponents();
+        List<Segment<K, V>> all = segments().allSorted(false);
+        if (all.isEmpty())
+            return null;
+        return all.get(0).descriptor;
+    }
+
     @SuppressWarnings("unused")
     ReferencedSegments<K, V> selectAndReference(Predicate<Segment<K,V>> selector)
     {
@@ -761,29 +810,28 @@ public class Journal<K, V> implements Shutdownable
 
     private void addNewActiveSegment(ActiveSegment<K, V> activeSegment)
     {
+        Invariants.require(isNotStopped());
         swapSegments(current -> current.withNewActiveSegment(activeSegment));
     }
 
     private void removeEmptySegment(ActiveSegment<K, V> activeSegment)
     {
+        Invariants.require(isNotStopped());
         swapSegments(current -> current.withoutEmptySegment(activeSegment));
     }
 
     private void replaceCompletedSegment(ActiveSegment<K, V> activeSegment, StaticSegment<K, V> staticSegment)
     {
+        Invariants.require(isNotStopped());
         swapSegments(current -> current.withCompletedSegment(activeSegment, staticSegment));
     }
 
     void replaceCompactedSegments(Collection<StaticSegment<K, V>> oldSegments, Collection<StaticSegment<K, V>> compactedSegments)
     {
+        Invariants.require(isNotStopped());
         swapSegments(current -> current.withCompactedSegments(oldSegments, compactedSegments));
     }
 
-    void selectSegmentToFlush(Collection<ActiveSegment<K, V>> into)
-    {
-        segments().selectActive(currentSegment.descriptor.timestamp, into);
-    }
-
     ActiveSegment<K, V> oldestActiveSegment()
     {
         ActiveSegment<K, V> current = currentSegment;
@@ -856,7 +904,7 @@ public class Journal<K, V> implements Shutdownable
             activeSegment.updateWrittenTo();
             activeSegment.fsync();
             activeSegment.persistComponents();
-            replaceCompletedSegment(activeSegment, StaticSegment.open(activeSegment.descriptor, keySupport));
+            replaceCompletedSegment(activeSegment, StaticSegment.open(activeSegment.descriptor, keySupport, FAIL));
             activeSegment.release(Journal.this);
         }
     }
@@ -892,8 +940,8 @@ public class Journal<K, V> implements Shutdownable
 
     static void validateCRC(CRC32 crc, int readCRC) throws Crc.InvalidCrc
     {
-        if (readCRC != (int) crc.getValue())
-            throw new Crc.InvalidCrc(readCRC, (int) crc.getValue());
+        if (readCRC != (int)crc.getValue())
+            throw new Crc.InvalidCrc(readCRC, (int)crc.getValue());
     }
 
     /*
@@ -950,6 +998,7 @@ public class Journal<K, V> implements Shutdownable
     @VisibleForTesting
     public void truncateForTesting()
     {
+        Invariants.require(isNotStopped());
         ActiveSegment<?, ?> discarding = currentSegment;
         if (!discarding.isEmpty()) // if there is no data in the segement then ignore it
         {
@@ -1158,12 +1207,16 @@ public class Journal<K, V> implements Shutdownable
         }
     }
 
-    enum State
+    public enum State
     {
         UNINITIALIZED,
-        INITIALIZING,
-        NORMAL,
-        SHUTDOWN,
-        TERMINATED
+        OPENING,
+        OPEN_READABLE,
+        STARTING,
+        WRITEABLE,
+        STOPPING,
+        STOPPED_READABLE,
+        CLOSING,
+        CLOSED
     }
 }

src/java/org/apache/cassandra/journal/Metadata.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy;
 import org.apache.cassandra.utils.Crc;
 
 import static org.apache.cassandra.journal.Journal.validateCRC;
@@ -132,11 +133,11 @@ final class Metadata
         }
     }
 
-    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport)
+    static <K> Metadata rebuild(Descriptor descriptor, KeySupport<K> keySupport, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
         int recordsCount = 0;
         int fsyncLimit = 0;
-        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, Integer.MAX_VALUE))
+        try (StaticSegment.SequentialReader<K> reader = StaticSegment.sequentialReader(descriptor, keySupport, Integer.MAX_VALUE, crcFailurePolicy))
         {
             while (reader.advance())
                 ++recordsCount;
@@ -152,9 +153,9 @@ final class Metadata
         return new Metadata(recordsCount, fsyncLimit);
     }
 
-    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport)
+    static <K> Metadata rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
-        Metadata metadata = rebuild(descriptor, keySupport);
+        Metadata metadata = rebuild(descriptor, keySupport, crcFailurePolicy);
         metadata.persist(descriptor);
         return metadata;
     }

src/java/org/apache/cassandra/journal/OnDiskIndex.java

@@ -33,6 +33,7 @@ import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.Crc;
 import org.apache.cassandra.utils.memory.MemoryUtil;
@@ -370,9 +371,9 @@ final class OnDiskIndex<K> extends Index<K>
         return keySupport.compareWithKeyAt(key, buffer, offset, descriptor.userVersion);
     }
 
-    static <K> OnDiskIndex<K> rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> OnDiskIndex<K> rebuildAndPersist(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
-        try (InMemoryIndex<K> index = InMemoryIndex.rebuild(descriptor, keySupport, fsyncedLimit))
+        try (InMemoryIndex<K> index = InMemoryIndex.rebuild(descriptor, keySupport, fsyncedLimit, crcFailurePolicy))
         {
             index.persist(descriptor);
         }

src/java/org/apache/cassandra/journal/Params.java

@@ -24,7 +24,14 @@ public interface Params
     enum FlushMode { BATCH, GROUP, PERIODIC }
 
     enum FailurePolicy { STOP, STOP_JOURNAL, IGNORE, ALLOW_UNSAFE_STARTUP, DIE }
-    enum ReplayMode { RESET, ALL, ONLY_NON_DURABLE }
+
+    enum RecoverableCrcFailurePolicy
+    {
+        FAIL,
+        IGNORE_ALL_ZERO_RECORDS,
+        IGNORE_CRC_ZERO_RECORDS,
+        IGNORE
+    }
 
     /**
      * @return maximum segment size
@@ -41,13 +48,13 @@ public interface Params
      */
     FailurePolicy failurePolicy();
 
+    RecoverableCrcFailurePolicy crcFailureOnRebuildPolicy();
+
     /**
      * @return journal flush (sync) mode
      */
     FlushMode flushMode();
 
-    ReplayMode replayMode();
-
     boolean enableCompaction();
 
     long compactionPeriod(TimeUnit units);

src/java/org/apache/cassandra/journal/Segment.java

@@ -44,7 +44,8 @@ public abstract class Segment<K, V> implements SelfRefCounted<Segment<K, V>>, Co
 
         public final void tidy()
         {
-            executor.execute(this);
+            if (executor != null) executor.execute(this);
+            else onUnreferenced();
         }
     }
 

src/java/org/apache/cassandra/journal/StaticSegment.java

@@ -32,11 +32,14 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy;
 import org.apache.cassandra.utils.Closeable;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.concurrent.Ref;
 import org.apache.cassandra.utils.memory.MemoryUtil;
 
+import static org.apache.cassandra.journal.Params.RecoverableCrcFailurePolicy.FAIL;
+
 /**
  * An immutable data segment that is no longer written to.
  * <p>
@@ -76,12 +79,12 @@ public final class StaticSegment<K, V> extends Segment<K, V>
      * @param descriptors descriptors of the segments to load
      * @return list of the loaded segments
      */
-    static <K, V> List<Segment<K, V>> open(Collection<Descriptor> descriptors, KeySupport<K> keySupport)
+    static <K, V> List<Segment<K, V>> open(Collection<Descriptor> descriptors, KeySupport<K> keySupport, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
         List<Segment<K, V>> segments = new ArrayList<>(descriptors.size());
         for (Descriptor descriptor : descriptors)
         {
-            StaticSegment<K, V> segment = open(descriptor, keySupport);
+            StaticSegment<K, V> segment = open(descriptor, keySupport, crcFailurePolicy);
             segments.add(segment);
         }
 
@@ -95,7 +98,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
      * @return the loaded segment
      */
     @SuppressWarnings({ "resource", "RedundantSuppression" })
-    static <K, V> StaticSegment<K, V> open(Descriptor descriptor, KeySupport<K> keySupport)
+    static <K, V> StaticSegment<K, V> open(Descriptor descriptor, KeySupport<K> keySupport, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
         if (!Component.DATA.existsFor(descriptor))
             throw new IllegalArgumentException("Data file for segment " + descriptor + " doesn't exist");
@@ -115,7 +118,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         }
 
         if (metadata == null)
-            metadata = Metadata.rebuildAndPersist(descriptor, keySupport);
+            metadata = Metadata.rebuildAndPersist(descriptor, keySupport, crcFailurePolicy);
 
         OnDiskIndex<K> index = null;
 
@@ -133,7 +136,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         }
 
         if (index == null)
-            index = OnDiskIndex.rebuildAndPersist(descriptor, keySupport, metadata.fsyncLimit());
+            index = OnDiskIndex.rebuildAndPersist(descriptor, keySupport, metadata.fsyncLimit(), crcFailurePolicy);
 
         try
         {
@@ -300,7 +303,7 @@ public final class StaticSegment<K, V> extends Segment<K, V>
      */
     public void forEachRecord(RecordConsumer<K> consumer)
     {
-        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, fsyncLimit))
+        try (SequentialReader<K> reader = sequentialReader(descriptor, keySupport, fsyncLimit, FAIL))
         {
             while (reader.advance())
             {
@@ -389,9 +392,9 @@ public final class StaticSegment<K, V> extends Segment<K, V>
         }
     }
 
-    static <K> SequentialReader<K> sequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+    static <K> SequentialReader<K> sequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit, RecoverableCrcFailurePolicy crcFailurePolicy)
     {
-        return new SequentialReader<>(descriptor, keySupport, fsyncedLimit);
+        return new SequentialReader<>(descriptor, keySupport, fsyncedLimit, crcFailurePolicy);
     }
 
     /**
@@ -405,11 +408,13 @@ public final class StaticSegment<K, V> extends Segment<K, V>
     static final class SequentialReader<K> extends Reader<K>
     {
         private final int fsyncedLimit; // exclusive
+        private final RecoverableCrcFailurePolicy crcFailurePolicy;
 
-        SequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit)
+        SequentialReader(Descriptor descriptor, KeySupport<K> keySupport, int fsyncedLimit, RecoverableCrcFailurePolicy crcFailurePolicy)
         {
             super(descriptor, keySupport);
             this.fsyncedLimit = fsyncedLimit;
+            this.crcFailurePolicy = crcFailurePolicy;
             if (fsyncedLimit < buffer.limit())
                 buffer.limit(fsyncedLimit);
         }
@@ -426,35 +431,45 @@ public final class StaticSegment<K, V> extends Segment<K, V>
 
         private boolean doAdvance()
         {
-            offset = buffer.position();
-            try
+            while (true)
             {
-                int length = EntrySerializer.tryRead(holder, keySupport, buffer.duplicate(), fsyncedLimit, descriptor.userVersion);
-                if (length < 0)
-                    return eof();
-                buffer.position(offset + length);
-            }
-            catch (EntrySerializer.MaybeRecoverableJournalError e)
-            {
-                logger.warn("Caught a recoverable journal error, skipping bytes", e);
-                int sizeMarker = buffer.getInt(offset);
-                if (e.knownLength <= Integer.BYTES || sizeMarker != offset + e.knownLength)
-                    throw new JournalReadError(descriptor, file,  e.getCause());
+                offset = buffer.position();
+                try
+                {
+                    int length = EntrySerializer.tryRead(holder, keySupport, buffer.duplicate(), fsyncedLimit, descriptor.userVersion);
+                    if (length < 0)
+                        return eof();
+                    buffer.position(offset + length);
+                    state = State.ADVANCED;
+                    return true;
+                }
+                catch (EntrySerializer.MaybeRecoverableJournalError e)
+                {
+                    int sizeMarker = buffer.getInt(offset);
+                    if (e.knownLength <= Integer.BYTES || sizeMarker != offset + e.knownLength)
+                        throw new JournalReadError(descriptor, file, e.getCause());
 
-                if (!areAllBytesZero(buffer, offset + Integer.BYTES, e.knownLength - Integer.BYTES))
-                    throw new JournalReadError(descriptor, file, e.getCause());
+                    switch (crcFailurePolicy)
+                    {
+                        case IGNORE: break;
+                        case IGNORE_ALL_ZERO_RECORDS:
+                            if (areAllBytesZero(buffer, offset + Integer.BYTES, e.knownLength - Integer.BYTES))
+                                break;
+                        case IGNORE_CRC_ZERO_RECORDS:
+                            if (e.readCrc == 0)
+                                break;
+                        case FAIL:
+                            throw new JournalReadError(descriptor, file, e.getCause());
+                    }
 
-                buffer.position(offset + e.knownLength);
-                // Recur here, as we anticipate a corrupt or incompletely written entry to be a very rare case.
-                return doAdvance();
+                    logger.warn("Caught a recoverable journal error, skipping bytes", e);
+                    buffer.position(offset + e.knownLength);
+                }
+                catch (IOException e)
+                {
+                    throw new JournalReadError(descriptor, file, e);
+                }
             }
-            catch (IOException e)
-            {
-                throw new JournalReadError(descriptor, file, e);
-            }
-
-            state = State.ADVANCED;
-            return true;
         }
 
         private void reset()

src/java/org/apache/cassandra/journal/ValueSerializer.java

@@ -31,4 +31,19 @@ public interface ValueSerializer<K, V>
      * redundant information in values, if it can be derived from keys.
      */
     V deserialize(K key, DataInputPlus in, int userVersion) throws IOException;
+
+    class Unsupported<K, V> implements ValueSerializer<K, V>
+    {
+        @Override
+        public void serialize(K key, V value, DataOutputPlus out, int userVersion)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public V deserialize(K key, DataInputPlus in, int userVersion)
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
 }

src/java/org/apache/cassandra/net/Verb.java

@@ -83,23 +83,18 @@ import org.apache.cassandra.schema.SchemaPushVerbHandler;
 import org.apache.cassandra.schema.SchemaVersionVerbHandler;
 import org.apache.cassandra.service.EchoVerbHandler;
 import org.apache.cassandra.service.SnapshotVerbHandler;
-import org.apache.cassandra.service.accord.AccordSerializers;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordSyncPropagator;
-import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
-import org.apache.cassandra.service.accord.FetchTopologies;
-import org.apache.cassandra.service.accord.WatermarkCollector;
 import org.apache.cassandra.service.accord.interop.AccordInteropApply;
 import org.apache.cassandra.service.accord.interop.AccordInteropRead;
 import org.apache.cassandra.service.accord.interop.AccordInteropReadRepair;
 import org.apache.cassandra.service.accord.interop.AccordInteropStableThenRead;
 import org.apache.cassandra.service.accord.serializers.AcceptSerializers;
+import org.apache.cassandra.service.accord.serializers.AccordSerializers;
 import org.apache.cassandra.service.accord.serializers.ApplySerializers;
 import org.apache.cassandra.service.accord.serializers.AwaitSerializers;
 import org.apache.cassandra.service.accord.serializers.BeginInvalidationSerializers;
 import org.apache.cassandra.service.accord.serializers.CheckStatusSerializers;
 import org.apache.cassandra.service.accord.serializers.CommitSerializers;
-import org.apache.cassandra.service.accord.serializers.EnumSerializer;
 import org.apache.cassandra.service.accord.serializers.FetchSerializers;
 import org.apache.cassandra.service.accord.serializers.GetDurableBeforeSerializers;
 import org.apache.cassandra.service.accord.serializers.GetEphmrlReadDepsSerializers;
@@ -110,7 +105,12 @@ import org.apache.cassandra.service.accord.serializers.PreacceptSerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializer;
 import org.apache.cassandra.service.accord.serializers.RecoverySerializers;
 import org.apache.cassandra.service.accord.serializers.SetDurableSerializers;
+import org.apache.cassandra.service.accord.serializers.SimpleReplySerializer;
 import org.apache.cassandra.service.accord.serializers.Version;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification;
+import org.apache.cassandra.service.accord.topology.FetchTopologies;
+import org.apache.cassandra.service.accord.topology.WatermarkCollector;
 import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState;
 import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.ConsensusKeyMigrationFinished;
 import org.apache.cassandra.service.paxos.Commit;
@@ -320,7 +320,7 @@ public enum Verb
     DATA_MOVEMENT_EXECUTED_REQ  (817, P1, rpcTimeout, MISC, () -> DataMovement.Status.serializer,   () -> DataMovements.instance,           DATA_MOVEMENT_EXECUTED_RSP  ),
 
     // accord
-    ACCORD_SIMPLE_RSP               (119, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(EnumSerializer.simpleReply),           AccordService::responseHandlerOrNoop                                           ),
+    ACCORD_SIMPLE_RSP               (119, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(SimpleReplySerializer.reply),          AccordService::responseHandlerOrNoop                                           ),
     ACCORD_PRE_ACCEPT_RSP           (120, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(PreacceptSerializers.reply),           AccordService::responseHandlerOrNoop                                           ),
     ACCORD_PRE_ACCEPT_REQ           (121, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(PreacceptSerializers.request),         AccordService::requestHandlerOrNoop, ACCORD_PRE_ACCEPT_RSP                     ),
     ACCORD_ACCEPT_RSP               (122, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(AcceptSerializers.reply),              AccordService::responseHandlerOrNoop                                           ),
@@ -357,10 +357,10 @@ public enum Verb
     ACCORD_GET_MAX_CONFLICT_REQ     (153, P2, readTimeout,  IMMEDIATE,          () -> accordEmbedded(GetMaxConflictSerializers.request),    AccordService::requestHandlerOrNoop, ACCORD_GET_MAX_CONFLICT_RSP),
     ACCORD_GET_DURABLE_BEFORE_RSP   (154, P2, rpcTimeout,   IMMEDIATE,          () -> accordEmbedded(GetDurableBeforeSerializers.reply),    AccordService::responseHandlerOrNoop                                           ),
     ACCORD_GET_DURABLE_BEFORE_REQ   (155, P2, rpcTimeout,   IMMEDIATE,          () -> accordEmbedded(GetDurableBeforeSerializers.request),  AccordService::requestHandlerOrNoop, ACCORD_GET_DURABLE_BEFORE_RSP             ),
-    ACCORD_SET_SHARD_DURABLE_REQ    (156, P2, rpcTimeout,   IMMEDIATE,          () -> accordEmbedded(SetDurableSerializers.shardDurable),   AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP                         ),
-    ACCORD_SET_GLOBALLY_DURABLE_REQ (157, P2, rpcTimeout,   IMMEDIATE,          () -> accordEmbedded(SetDurableSerializers.globallyDurable),AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP                         ),
+    ACCORD_SET_SHARD_DURABLE_REQ    (156, P2, rpcTimeout,   MISC,               () -> accordEmbedded(SetDurableSerializers.shardDurable),   AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP                         ),
+    ACCORD_SET_GLOBALLY_DURABLE_REQ (157, P2, rpcTimeout,   MISC,               () -> accordEmbedded(SetDurableSerializers.globallyDurable),AccordService::requestHandlerOrNoop, ACCORD_SIMPLE_RSP                         ),
 
-    ACCORD_SYNC_NOTIFY_RSP          (158, P2, writeTimeout, MISC,               () -> accordEmbedded(EnumSerializer.simpleReply),           RESPONSE_HANDLER),
+    ACCORD_SYNC_NOTIFY_RSP          (158, P2, writeTimeout, MISC,               () -> accordEmbedded(SimpleReplySerializer.reply),          RESPONSE_HANDLER),
     ACCORD_SYNC_NOTIFY_REQ          (159, P2, writeTimeout, MISC,               () -> accordEmbedded(Notification.serializer),              () -> AccordSyncPropagator.verbHandler,       ACCORD_SYNC_NOTIFY_RSP             ),
 
     CONSENSUS_KEY_MIGRATION         (160, P1, writeTimeout,  MISC,              () -> accordEmbedded(ConsensusKeyMigrationFinished.serializer),() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler),

src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java

@@ -35,7 +35,7 @@ import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.exceptions.CasWriteTimeoutException;
 import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MetadataSnapshots;

src/java/org/apache/cassandra/schema/KeyspaceParams.java

@@ -28,7 +28,7 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;

src/java/org/apache/cassandra/schema/SchemaKeyspace.java

@@ -84,7 +84,7 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 import org.apache.cassandra.utils.ByteBufferUtil;

src/java/org/apache/cassandra/schema/TableId.java

@@ -188,9 +188,14 @@ public final class TableId implements Comparable<TableId>
 
     @Override
     public String toString()
+    {
+        return toShortString("tid:");
+    }
+
+    public String toShortString(String prefix)
     {
         if (msb == MAGIC)
-            return "tid:" + Long.toHexString(lsb);
+            return prefix + Long.toHexString(lsb);
         return asUUID().toString();
     }
 

src/java/org/apache/cassandra/schema/TableMetadata.java

@@ -76,7 +76,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;

src/java/org/apache/cassandra/schema/TableParams.java

@@ -34,7 +34,7 @@ import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.topology.FastPathStrategy;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
 import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy;

src/java/org/apache/cassandra/service/StorageService.java

@@ -163,7 +163,6 @@ import org.apache.cassandra.metrics.SamplingManager;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.RepairCoordinator;
-import org.apache.cassandra.repair.RepairParallelism;
 import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.repair.autorepair.AutoRepair;
 import org.apache.cassandra.repair.messages.RepairOption;
@@ -178,7 +177,6 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.schema.ViewMetadata;
-import org.apache.cassandra.service.accord.AccordKeyspace.AccordColumnFamilyStores;
 import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget;
@@ -190,7 +188,6 @@ import org.apache.cassandra.service.paxos.PaxosState;
 import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupLocalCoordinator;
 import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState;
 import org.apache.cassandra.service.snapshot.SnapshotManager;
-import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.streaming.StreamManager;
 import org.apache.cassandra.streaming.StreamResultFuture;
 import org.apache.cassandra.streaming.StreamState;
@@ -249,6 +246,7 @@ import static java.util.Arrays.asList;
 import static java.util.Arrays.stream;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
@@ -260,7 +258,6 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPA
 import static org.apache.cassandra.config.CassandraRelevantProperties.PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_RETRY_DELAY_SECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.REPLACE_ADDRESS_FIRST_BOOT;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_WRITE_SURVEY;
-import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.INTERNALLY_FORCED;
 import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName;
 import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily;
 import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
@@ -3189,27 +3186,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return new FutureTask<>(task);
     }
 
-    public RepairCoordinator repairAccordKeyspace(String keyspace, Collection<Range<Token>> ranges)
+    public RepairCoordinator newRepairCoordinator(String keyspace, RepairOption options)
     {
         int cmd = nextRepairCommand.incrementAndGet();
-        RepairOption options = new RepairOption(RepairParallelism.PARALLEL, // parallelism
-                                                false,                       // primaryRange
-                                                false,                      // incremental
-                                                false,                      // trace
-                                                5,                          // jobThreads
-                                                ranges,                     // ranges
-                                                true,                       // pullRepair
-                                                true,                       // forceRepair
-                                                PreviewKind.NONE,           // previewKind
-                                                false,                      // optimiseStreams
-                                                true,                       // ignoreUnreplicatedKeyspaces
-                                                true,                       // repairData
-                                                false,                      // repairPaxos
-                                                true,                       // dontPurgeTombstones
-                                                false,                      // repairAccord
-                                                false                       // permit no quorum
-        );
-
         return new RepairCoordinator(this, cmd, options, keyspace);
     }
 
@@ -3889,7 +3868,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             }
 
             if (AccordService.isSetupOrStarting())
-                AccordService.unsafeInstance().markShuttingDown();
+                AccordService.unsafeInstance().stop();
 
             // In-progress writes originating here could generate hints to be written,
             // which is currently scheduled on the mutation stage. So shut down MessagingService
@@ -3907,12 +3886,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
             if (AccordService.isSetupOrStarting())
             {
-                logger.info("Flushing Accord caches");
-                if (!AccordService.unsafeInstance().flushCaches().awaitUninterruptibly(1, MINUTES))
-                    logger.error("Could not flush Accord caches promptly");
-                if (AccordColumnFamilyStores.commandsForKey != null)
-                    AccordColumnFamilyStores.commandsForKey.forceBlockingFlush(INTERNALLY_FORCED);
-                AccordService.unsafeInstance().shutdownAndWait(1, MINUTES);
+                try
+                {
+                    AccordService.unsafeInstance().shutdownAndWait(DatabaseDescriptor.getAccord().shutdown_grace_period.toDuration().toNanos(), NANOSECONDS);
+                }
+                catch (Throwable t)
+                {
+                    logger.error("AccordService exception shutting down", t);
+                }
             }
 
             // ScheduledExecutors shuts down after MessagingService, as MessagingService may issue tasks to it.

src/java/org/apache/cassandra/service/accord/AccordCache.java

@@ -68,6 +68,8 @@ import org.apache.cassandra.service.accord.AccordCache.Adapter.Shrink;
 import org.apache.cassandra.service.accord.AccordCacheEntry.LoadExecutor;
 import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
 import org.apache.cassandra.service.accord.events.CacheEvents;
+import org.apache.cassandra.service.accord.journal.CommandChangeWriter;
+import org.apache.cassandra.service.accord.journal.CommandChanges;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.service.accord.serializers.Version;
 import org.apache.cassandra.utils.NoSpamLogger;
@@ -532,7 +534,7 @@ public class AccordCache implements CacheSize
 
                 AccordCacheEntry<K, V> node = cache.get(key);
 
-                require(!safeRef.invalidated());
+                require(!safeRef.isUnsafe());
                 require(safeRef.global() != null, "safeRef node is null for %s", key);
                 require(safeRef.global() == node, "safeRef node not in map: %s != %s", safeRef.global(), node);
                 require(node.references() > 0, "references (%d) are zero for %s (%s)", node.references(), key, node);
@@ -561,7 +563,7 @@ public class AccordCache implements CacheSize
                 {
                     evict = node.is(LOADED) && node.isNull();
                 }
-                safeRef.invalidate();
+                safeRef.markUnsafe();
 
                 if (node.decrement() == 0)
                 {
@@ -1298,7 +1300,7 @@ public class AccordCache implements CacheSize
 
             try
             {
-                return AccordJournal.asSerializedChange(null, value, Version.LATEST);
+                return CommandChangeWriter.asSerializedChange(null, value, Version.LATEST);
             }
             catch (IOException e)
             {
@@ -1314,7 +1316,7 @@ public class AccordCache implements CacheSize
                 Invariants.expect(value.saveStatus().compareTo(SaveStatus.ReadyToExecute) < 0);
 
             // TODO (expected): improve heuristics and consider transaction size
-            if (SHRINK_WITHOUT_LOCK < 0 || value.partialDeps() == null || value.partialDeps().txnIds().size() < SHRINK_WITHOUT_LOCK)
+            if (SHRINK_WITHOUT_LOCK <= 0 || value.partialDeps() == null || value.partialDeps().txnIds().size() <= SHRINK_WITHOUT_LOCK)
                 return Shrink.DONE;
 
             return Shrink.PERFORM_WITHOUT_LOCK;
@@ -1323,7 +1325,7 @@ public class AccordCache implements CacheSize
         @Override
         public @Nullable Command inflate(AccordCommandStore commandStore, TxnId key, Object serialized)
         {
-            AccordJournal.Builder builder = new AccordJournal.Builder(key);
+            CommandChanges builder = new CommandChanges(key);
             ByteBuffer buffer = (ByteBuffer) serialized;
             buffer.mark();
             try (DataInputBuffer buf = new DataInputBuffer(buffer, false))

src/java/org/apache/cassandra/service/accord/AccordCommandStore.java

@@ -18,17 +18,20 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Objects;
 import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import java.util.concurrent.locks.Lock;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntFunction;
+import java.util.function.Predicate;
 
 import javax.annotation.Nullable;
 
@@ -42,18 +45,29 @@ import accord.api.AsyncExecutor;
 import accord.api.DataStore;
 import accord.api.Journal;
 import accord.api.LocalListeners;
+import accord.api.LocalListeners.TxnListener;
 import accord.api.ProgressLog;
 import accord.api.RoutingKey;
 import accord.impl.AbstractReplayer;
+import accord.impl.AbstractReplayer.Mode;
 import accord.impl.AbstractSafeCommandStore.CommandStoreCaches;
+import accord.impl.DefaultLocalListeners;
 import accord.impl.progresslog.DefaultProgressLog;
+import accord.impl.progresslog.TxnState;
 import accord.local.Command;
 import accord.local.CommandStore;
 import accord.local.CommandStores;
 import accord.local.CommandSummaries;
+import accord.local.MaxConflicts;
+import accord.local.MaxDecidedRX;
 import accord.local.NodeCommandStoreService;
 import accord.local.PreLoadContext;
+import accord.local.PreLoadContext.Empty;
 import accord.local.RedundantBefore;
+import accord.local.RedundantBefore.Bounds;
+import accord.local.RedundantStatus.Property;
+import accord.local.RedundantStatus.SomeStatus;
+import accord.local.RejectBefore;
 import accord.local.SafeCommandStore;
 import accord.local.cfk.CommandsForKey;
 import accord.primitives.PartialTxn;
@@ -61,32 +75,60 @@ import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.RoutableKey;
 import accord.primitives.Route;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
 import accord.utils.UnhandledEnum;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults.CountingResult;
 
 import org.apache.cassandra.config.AccordSpec;
+import org.apache.cassandra.config.AccordSpec.JournalSpec.ReplayMode;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.journal.Descriptor;
 import org.apache.cassandra.metrics.LogLinearDecayingHistograms;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable;
 import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
 import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo;
 import org.apache.cassandra.service.accord.api.TokenKey;
+import org.apache.cassandra.service.accord.journal.AccordJournal;
+import org.apache.cassandra.service.accord.journal.JournalRangeIndex;
 import org.apache.cassandra.service.accord.txn.TxnRead;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.concurrent.Condition;
 
 import static accord.api.Journal.CommandUpdate;
 import static accord.api.Journal.FieldUpdates;
+import static accord.impl.progresslog.DefaultProgressLog.ModeFlag.CATCH_UP;
+import static accord.local.RedundantStatus.Property.LOCALLY_APPLIED;
+import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE;
+import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE;
+import static accord.local.RedundantStatus.SomeStatus.LOCALLY_DURABLE_TO_COMMAND_STORE_ONLY;
+import static accord.local.RedundantStatus.SomeStatus.LOCALLY_DURABLE_TO_DATA_STORE_ONLY;
+import static accord.primitives.Status.Durability.HasOutcome.Universal;
 import static accord.utils.Invariants.require;
-import static org.apache.cassandra.journal.Params.ReplayMode.ONLY_NON_DURABLE;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccord;
+import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readList;
+import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readOne;
+import static org.apache.cassandra.io.util.CompressedFrameDataOutputPlus.writeList;
+import static org.apache.cassandra.io.util.CompressedFrameDataOutputPlus.writeOne;
+import static org.apache.cassandra.service.accord.journal.ReplayMarkers.saveDirectory;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.maxConflicts;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.maxDecidedRX;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.progressLogState;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.redundantBefore;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.rejectBefore;
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.txnListener;
 
 public class AccordCommandStore extends CommandStore
 {
@@ -152,20 +194,35 @@ public class AccordCommandStore extends CommandStore
         }
     }
 
+    static class Termination extends Condition.Sync
+    {
+        private boolean commandStoreFlushed;
+        private boolean dataStoreFlushed;
+        private boolean isReadyToTerminate()
+        {
+            return commandStoreFlushed && dataStoreFlushed;
+        }
+    }
+
     static final AtomicReferenceFieldUpdater<AccordCommandStore, SafeRedundantBefore> safeRedundantBeforeUpdater
         = AtomicReferenceFieldUpdater.newUpdater(AccordCommandStore.class, SafeRedundantBefore.class, "safeRedundantBefore");
+    static final AtomicReferenceFieldUpdater<AccordCommandStore, Termination> terminatedUpdater
+        = AtomicReferenceFieldUpdater.newUpdater(AccordCommandStore.class, Termination.class, "terminated");
     static final AtomicLong nextSafeRedundantBeforeTicket = new AtomicLong();
 
+    private static final AtomicLong lastSystemTimestampMicros = new AtomicLong();
+
     public final String loggingId;
     public final Journal journal;
     private final AccordExecutor sharedExecutor;
     final AccordExecutor.SequentialExecutor exclusiveExecutor;
     private final ExclusiveCaches caches;
-    private long lastSystemTimestampMicros = Long.MIN_VALUE;
     private final RangeIndex rangeIndex;
     private final TableId tableId;
     private TableMetadataRef metadata;
+
     volatile SafeRedundantBefore safeRedundantBefore;
+    volatile Termination terminated;
 
     private AccordSafeCommandStore current;
     LogLinearDecayingHistograms.Buffer metricsBuffer;
@@ -193,12 +250,12 @@ public class AccordCommandStore extends CommandStore
         {
             commands = exclusive.commands.newInstance(this);
             commandsForKey = exclusive.commandsForKey.newInstance(this);
-            this.caches = new ExclusiveCaches(sharedExecutor.lock, exclusive.global, commands, commandsForKey);
+            this.caches = new ExclusiveCaches(sharedExecutor.unsafeLock(), exclusive.global, commands, commandsForKey);
         }
 
         this.exclusiveExecutor = sharedExecutor.executor(id);
         {
-            AccordSpec.RangeIndexMode mode = DatabaseDescriptor.getAccord().range_index_mode;
+            AccordSpec.RangeIndexMode mode = getAccord().range_index_mode;
             switch (mode)
             {
                 default: throw new UnhandledEnum(mode);
@@ -329,8 +386,8 @@ public class AccordCommandStore extends CommandStore
         if (cfk == null)
             return null;
         RedundantBefore.QuickBounds bounds = safeGetRedundantBefore().get(key);
-        if (bounds == null)
-            return cfk; // TODO (required): I don't think this should be possible? but we hit it on some test
+        if (!Invariants.expect(bounds != null, "No RedundantBefore information found when loading key %s", key))
+            return cfk;
         return cfk.withGcBeforeAtLeast(bounds.gcBefore, false);
     }
 
@@ -351,8 +408,7 @@ public class AccordCommandStore extends CommandStore
 
     public long nextSystemTimestampMicros()
     {
-        lastSystemTimestampMicros = Math.max(TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()), lastSystemTimestampMicros + 1);
-        return lastSystemTimestampMicros;
+        return lastSystemTimestampMicros.accumulateAndGet(node.now(), (a, b) -> Math.max(a + 1, b));
     }
     @Override
     public <T> AsyncChain<T> chain(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
@@ -416,6 +472,70 @@ public class AccordCommandStore extends CommandStore
     @Override
     public void shutdown()
     {
+        shutdownAsync();
+    }
+
+    public AsyncResult<Void> shutdownAsync()
+    {
+        terminatedUpdater.compareAndSet(this, null, new Termination());
+        progressLog.stop();
+        return execute((Empty)() -> "Shutdown", safeStore -> {
+            exclusiveExecutor.stop();
+            logger.info("{} stopping. Durably applied: {}, waiting: {}", this,
+                        DurablyAppliedTo.summarise(safeStore.redundantBefore(), DurablyAppliedTo::isDone),
+                        DurablyAppliedTo.summarise(safeStore.redundantBefore(), DurablyAppliedTo::isNotDone));
+            this.ensureDurable(null, ReportDurable.commandStoreFlush());
+            dataStore.ensureDurable(this, RedundantBefore.EMPTY, ReportDurable.DATA_STORE_FLUSH);
+        });
+    }
+
+    @Override
+    public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges durableRanges, Status.Durability.HasOutcome durability)
+    {
+        super.markShardDurable(safeStore, globalSyncId, durableRanges, durability);
+        if (durability == Universal)
+            rangeIndex.prune(globalSyncId, durableRanges, safeStore.redundantBefore());
+    }
+
+    @Override
+    protected void markExclusiveSyncPointLocallyApplied(SafeCommandStore safeStore, TxnId syncId, Ranges ranges, SaveStatus prevStatus)
+    {
+        super.markExclusiveSyncPointLocallyApplied(safeStore, syncId, ranges, prevStatus);
+        rangeIndex.prune(syncId, ranges, safeStore.redundantBefore());
+    }
+
+    void maybeTerminated(boolean setCommandStoreDurable, boolean setDataStoreDurable)
+    {
+        if (terminated != null)
+        {
+            if (setCommandStoreDurable) terminated.commandStoreFlushed = true;
+            if (setDataStoreDurable) terminated.dataStoreFlushed = true;
+            if (terminated.isReadyToTerminate())
+            {
+                Invariants.require(exclusiveExecutor.stopped());
+                boolean syncPointsDurable = unsafeGetRedundantBefore().foldl((b, v, p2, p3) -> {
+                    return v && (b == null || b.maxBound(LOCALLY_APPLIED).compareTo(b.maxBoundBoth(LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE)) <= 0);
+                }, true, null, null, ignore -> false);
+
+                if (!syncPointsDurable)
+                    logger.error("{} has flushed command and data stores, but sync points recorded in RedundantBefore are not durable: {}", this, DurablyAppliedTo.summarise(unsafeGetRedundantBefore()));
+
+                exclusiveExecutor.terminate();
+                terminated.signalAll();
+            }
+        }
+    }
+
+    public boolean awaitTerminationUntil(long deadlineNanos)
+    {
+        if (terminated == null)
+            throw new IllegalStateException("Not shutdown");
+        return terminated.awaitUntilThrowUncheckedOnInterrupt(deadlineNanos);
+    }
+
+    public boolean isTerminated()
+    {
+        return terminated != null && terminated.isSignalled();
     }
 
     public void appendCommands(List<CommandUpdate> diffs, Runnable onFlush)
@@ -481,44 +601,81 @@ public class AccordCommandStore extends CommandStore
         return safeRedundantBefore.redundantBefore;
     }
 
-    public AccordCommandStoreReplayer replayer()
+    @Override
+    public AccordCommandStoreReplayer replayer(Mode mode)
     {
-        boolean replayOnlyNonDurable = true;
-        if (journal instanceof AccordJournal)
-            replayOnlyNonDurable = ((AccordJournal)journal).configuration().replayMode() == ONLY_NON_DURABLE;
-        return new AccordCommandStoreReplayer(this, replayOnlyNonDurable);
+        ReplayMode replayMode = getAccord().journal.replay;
+        return new AccordCommandStoreReplayer(this, mode);
     }
-
+    
     static final AtomicLong nextDurabilityLoggingId = new AtomicLong();
+
     @Override
     protected void ensureDurable(Ranges ranges, RedundantBefore onCommandStoreDurable)
     {
-        if (node().isReplaying())
+        ensureDurable(ranges, ReportDurable.of(onCommandStoreDurable));
+    }
+
+    protected void ensureDurable()
+    {
+        RedundantBefore forCommandStore = nonDurable(unsafeGetRedundantBefore(), LOCALLY_DURABLE_TO_COMMAND_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE_ONLY);
+        RedundantBefore forDataStore = nonDurable(unsafeGetRedundantBefore(), LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_DATA_STORE_ONLY);
+        this.ensureDurable(forCommandStore.ranges(Objects::nonNull), forCommandStore);
+        dataStore.ensureDurable(this, forDataStore, 0);
+    }
+
+    private RedundantBefore nonDurable(RedundantBefore redundantBefore, Property durableProperty, SomeStatus durableStatus)
+    {
+        return redundantBefore.map(b -> {
+            if (b == null)
+                return null;
+
+            TxnId applied = b.maxBound(LOCALLY_APPLIED);
+            TxnId durable = b.maxBound(durableProperty);
+            if (applied.compareTo(durable) <= 0)
+                return null;
+            return Bounds.create(b.range, b.maxBound(LOCALLY_APPLIED), durableStatus, null);
+        });
+    }
+
+    protected void ensureDurable(@Nullable Ranges ranges, ReportDurable onCommandStoreDurable)
+    {
+        if (node().isReplaying() && onCommandStoreDurable.flags == 0 && unsafeGetRedundantBefore().isAtLeast(onCommandStoreDurable.redundantBefore))
             return;
 
         long reportId = nextDurabilityLoggingId.incrementAndGet();
-        logger.debug("{} awaiting local metadata durability for {} ({})", this, ranges, reportId);
+        logger.debug("{} durability: ensuring for {} ({})", this, onCommandStoreDurable, reportId);
         executor().afterSubmittedAndConsequences(() -> {
-            logger.debug("{}: saving intersecting keys ({})", this, reportId);
+            logger.debug("{} durability: saving intersecting keys ({})", this, reportId);
             class Ready extends CountingResult implements Runnable
             {
                 public Ready() { super(1); }
                 @Override public void run() { decrement(); }
+
+                void maybeFlush(ExclusiveCaches caches, AccordCacheEntry<RoutingKey, CommandsForKey> e)
+                {
+                    if (e.isModified())
+                    {
+                        increment();
+                        caches.global().saveWhenReadyExclusive(e, this);
+                    }
+                }
             }
 
             Ready ready = new Ready();
             try (ExclusiveCaches caches = lockCaches())
             {
-                for (Range range : ranges)
+                if (ranges == null)
                 {
-                    for (RoutingKey k : caches.commandsForKeys().keysBetween(range.start(), range.startInclusive(), range.end(), range.endInclusive()))
+                    for (AccordCacheEntry<RoutingKey, CommandsForKey> e : caches.commandsForKeys())
+                        ready.maybeFlush(caches, e);
+                }
+                else
+                {
+                    for (Range range : ranges)
                     {
-                        AccordCacheEntry<RoutingKey, CommandsForKey> e = caches.commandsForKeys().getUnsafe(k);
-                        if (e.isModified())
-                        {
-                            ready.increment();
-                            caches.global().saveWhenReadyExclusive(e, ready);
-                        }
+                        for (RoutingKey k : caches.commandsForKeys().keysBetween(range.start(), range.startInclusive(), range.end(), range.endInclusive()))
+                            ready.maybeFlush(caches, caches.commandsForKeys().getUnsafe(k));
                     }
                 }
             }
@@ -526,19 +683,14 @@ public class AccordCommandStore extends CommandStore
             ready.invoke((success, fail) -> {
                 if (fail != null)
                 {
-                    logger.error("{}: failed to ensure durability of {} ({})", this, ranges, reportId, fail);
+                    logger.error("{} failed to ensure durability of {} ({})", this, ranges, reportId, fail);
                 }
                 else
                 {
-                    logger.debug("{}: waiting for CommandsForKey to flush ({})", this, reportId);
+                    logger.debug("{} waiting for CommandsForKey to flush ({})", this, reportId);
                     ColumnFamilyStore cfs = AccordKeyspace.AccordColumnFamilyStores.commandsForKey;
 
-                    AccordDurableOnFlush onFlush = null;
-                    while (onFlush == null)
-                        onFlush = cfs.getCurrentMemtable().ensureFlushListener(AccordDataStore.FlushListenerKey.KEY, AccordDurableOnFlush::new);
-
-                    if (!onFlush.add(id, onCommandStoreDurable))
-                        AccordDurableOnFlush.notify(cfs.metadata(), this, onCommandStoreDurable);
+                    AccordDurableOnFlush.notifyOnDurable(cfs, this, onCommandStoreDurable);
                 }
             });
             ready.decrement();
@@ -561,26 +713,25 @@ public class AccordCommandStore extends CommandStore
     public static class AccordCommandStoreReplayer extends AbstractReplayer
     {
         private final AccordCommandStore commandStore;
-        private final boolean onlyNonDurable;
 
-        private AccordCommandStoreReplayer(AccordCommandStore commandStore, boolean onlyNonDurable)
+        private AccordCommandStoreReplayer(AccordCommandStore commandStore, Mode mode)
         {
-            super(commandStore, null);
+            super(commandStore, mode, null);
             this.commandStore = commandStore;
-            this.onlyNonDurable = onlyNonDurable;
         }
 
         @Override
         public AsyncChain<Route> replay(TxnId txnId)
         {
-            if (onlyNonDurable && !maybeShouldReplay(txnId))
+            if (!maybeShouldReplay(txnId))
                 return AsyncChains.success(null);
 
             return commandStore.chain(PreLoadContext.contextFor(txnId, "Replay"), safeStore -> {
-                if (onlyNonDurable && !shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants()))
+                Replay replay = shouldReplay(txnId, safeStore.unsafeGet(txnId).current().participants());
+                if (replay == Replay.NONE)
                     return null;
 
-                initialiseState(safeStore, txnId);
+                replay(safeStore, txnId, replay);
                 return safeStore.unsafeGet(txnId).current().route();
             });
         }
@@ -622,6 +773,141 @@ public class AccordCommandStore extends CommandStore
             loadRangesForEpoch(rangesForEpoch);
     }
 
+    AsyncChain<Boolean> saveState(Descriptor descriptor)
+    {
+        return chain((AccordExecutor.Unterminatable)() -> "Save State", safeStore -> {
+            File storeDir = storeSaveDir();
+            {
+                File[] tmpDirs = listTmpSaveDirs(storeDir);
+                if (tmpDirs != null)
+                {
+                    logger.info("Cleaning up incomplete save points: {}", Arrays.toString(tmpDirs));
+                    for (File dir : tmpDirs)
+                        dir.tryDeleteRecursive();
+                }
+            }
+
+            RedundantBefore validateRedundantBefore = journal.loadRedundantBefore(id);
+            Invariants.expect(validateRedundantBefore.equals(unsafeGetRedundantBefore()), "Journal RedundantBefore does not match in memory: %s != %s", validateRedundantBefore, unsafeGetRedundantBefore());
+
+            File[] sortedSaveDirs = listSortedSaveDirs(storeDir);
+            File tmpSaveDir = new File(storeDir, "tmp" + descriptor.timestamp);
+            File saveDir = new File(storeDir, "" + descriptor.timestamp);
+            if (sortedSaveDirs != null && Long.parseLong(sortedSaveDirs[sortedSaveDirs.length - 1].name()) >= descriptor.timestamp)
+            {
+                logger.error("There already exists a save point {} >= {}; aborting.", sortedSaveDirs[sortedSaveDirs.length - 1].name(), descriptor.timestamp);
+                return false;
+            }
+
+            try
+            {
+                logger.info("{} saving state to {}", this, saveDir);
+                tmpSaveDir.createDirectoriesIfNotExists();
+                writeOne(new File(tmpSaveDir, "max_decidedrx"), unsafeGetMaxDecidedRX(), maxDecidedRX);
+                writeOne(new File(tmpSaveDir, "max_conflicts"), unsafeGetMaxConflicts(), maxConflicts);
+                writeOne(new File(tmpSaveDir, "reject_before"), unsafeGetRejectBefore(), rejectBefore);
+                writeList(new File(tmpSaveDir, "listeners"), ((DefaultLocalListeners)listeners).snapshot(), txnListener);
+                writeList(new File(tmpSaveDir, "progress_log"), ((DefaultProgressLog)progressLog).snapshot(), progressLogState);
+                rangeIndex.save(new File(tmpSaveDir, "range_index"));
+                writeOne(new File(tmpSaveDir, "redundant_before"), unsafeGetRedundantBefore(), redundantBefore);
+                tmpSaveDir.move(saveDir);
+            }
+            catch (Throwable t)
+            {
+                logger.error("{} failed to save replay state {}", this, saveDir, t);
+                tmpSaveDir.tryDeleteRecursive();
+                saveDir.tryDeleteRecursive();
+                return false;
+            }
+
+            if (sortedSaveDirs != null)
+            {
+                int delete = (sortedSaveDirs.length + 1) - getAccord().journal.retainSavePoints;
+                if (delete > 0)
+                {
+                    sortedSaveDirs = Arrays.copyOf(sortedSaveDirs, delete);
+                    logger.debug("Deleting old save points: {}", Arrays.toString(sortedSaveDirs));
+                    for (File dir : sortedSaveDirs)
+                        dir.tryDeleteRecursive();
+                }
+            }
+
+            return true;
+        });
+    }
+
+    private File storeSaveDir()
+    {
+        return new File(saveDirectory(), String.format("%s_%d", tableId().toShortString(""), id()));
+    }
+
+    private static File[] listSortedSaveDirs(File storeDir)
+    {
+        File[] savePoints = storeDir.tryList(f -> f.isDirectory() && f.name().matches("[0-9]+"));
+        if (savePoints == null || savePoints.length == 0)
+            return null;
+
+        Arrays.sort(savePoints, Comparator.comparingLong(f -> Long.parseLong(f.name())));
+        return savePoints;
+    }
+
+    private static File[] listTmpSaveDirs(File storeDir)
+    {
+        File[] tmpDirs = storeDir.tryList(f -> f.isDirectory() && f.name().matches("tmp[0-9]+"));
+        if (tmpDirs == null || tmpDirs.length == 0)
+            return null;
+        return tmpDirs;
+    }
+
+    AsyncChain<Map.Entry<Integer, Long>> restoreState()
+    {
+        return chain((Empty)() -> "Restore State", safeStore -> {
+            File storeDir = storeSaveDir();
+            File[] savePoints = listSortedSaveDirs(storeDir);
+            if (savePoints == null)
+            {
+                logger.info("{} no save points found at {}", this, storeDir);
+                return null;
+            }
+
+            File savePoint = savePoints[savePoints.length - 1];
+            long segment = Long.parseLong(savePoint.name());
+            MaxDecidedRX mxd; MaxConflicts mxc; RejectBefore rjb;
+            List<TxnListener> dll; List<TxnState> dpl; Object rgi;
+            RedundantBefore rdb;
+            try
+            {
+                logger.info("{} loading state from {}", this, savePoint);
+                mxd = readOne(new File(savePoint, "max_decidedrx"), maxDecidedRX);
+                mxc = readOne(new File(savePoint, "max_conflicts"), maxConflicts);
+                rjb = readOne(new File(savePoint, "reject_before"), rejectBefore);
+                dll = readList(new File(savePoint, "listeners"), txnListener);
+                dpl = readList(new File(savePoint, "progress_log"), progressLogState);
+                rgi = rangeIndex.load(new File(savePoint, "range_index"));
+                rdb = readOne(new File(savePoint, "redundant_before"), redundantBefore);
+            }
+            catch (Throwable t)
+            {
+                logger.warn("{} could not replay save point {}", this, savePoint, t);
+                return null;
+            }
+
+            if (journal instanceof AccordJournal && ((AccordJournal)journal).maxDescriptor() <= segment)
+                Invariants.expect(rdb.equals(unsafeGetRedundantBefore()));
+
+            rangeIndex.restore(rgi);
+            unsafeSetMaxDecidedRX(mxd);
+            unsafeSetMaxConflicts(mxc);
+            unsafeSetRejectBefore(rjb);
+            ((DefaultLocalListeners) listeners).restore(dll);
+            boolean unsetCatchup = ((DefaultProgressLog) progressLog).setModeExclusive(safeStore, CATCH_UP);
+            ((DefaultProgressLog) progressLog).restore(safeStore, dpl);
+            if (unsetCatchup)
+                ((DefaultProgressLog) progressLog).unsetModeExclusive(CATCH_UP);
+            return Map.entry(id, segment + 1);
+        });
+    }
+
     // TODO (expected): handle journal failures, and consider how we handle partial failures.
     //  Very likely we will not be able to safely or cleanly handle partial failures of this logic, but decide and document.
     // TODO (desired): consider merging with PersistentField? This version is cheaper to manage which may be preferable at the CommandStore level.
@@ -663,7 +949,74 @@ public class AccordCommandStore extends CommandStore
         if (metadata != null)
             sb.append(metadata).append('|');
         sb.append(tableId);
-        sb.append('|').append(id).append(',').append(node.id().id).append(']');
+        sb.append('|')
+          .append(id).append(',')
+          .append(executor().executorId).append(',')
+          .append(node.id().id)
+          .append(']');
         return sb.toString();
     }
+
+    public static class DurablyAppliedTo
+    {
+        final TxnId journal, commandStore, dataStore;
+
+        public DurablyAppliedTo(Bounds bounds)
+        {
+            this(bounds.maxBound(LOCALLY_APPLIED), bounds.maxBound(LOCALLY_DURABLE_TO_COMMAND_STORE), bounds.maxBound(LOCALLY_DURABLE_TO_DATA_STORE));
+        }
+
+        public DurablyAppliedTo(TxnId journal, TxnId commandStore, TxnId dataStore)
+        {
+            this.journal = journal;
+            this.commandStore = commandStore;
+            this.dataStore = dataStore;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "journal:" + journal + ", commandStore:" + commandStore + ", dataStore:" + dataStore;
+        }
+
+        public boolean isDone()
+        {
+            return journal.compareTo(TxnId.min(commandStore, dataStore)) <= 0;
+        }
+
+        public boolean isNotDone()
+        {
+            return !isDone();
+        }
+
+        @Override
+        public boolean equals(Object that)
+        {
+            return that instanceof DurablyAppliedTo && equals((DurablyAppliedTo) that);
+        }
+
+        public boolean equals(DurablyAppliedTo that)
+        {
+            return this.journal.equals(that.journal)
+                   && this.commandStore.equals(that.commandStore)
+                   && this.dataStore.equals(that.dataStore);
+        }
+
+        public static ReducingRangeMap<DurablyAppliedTo> summarise(RedundantBefore redundantBefore)
+        {
+            return redundantBefore.map(b -> b == null ? null : new DurablyAppliedTo(b), DurablyAppliedTo[]::new);
+        }
+
+        public static ReducingRangeMap<DurablyAppliedTo> summarise(RedundantBefore redundantBefore, Predicate<DurablyAppliedTo> include)
+        {
+            return redundantBefore.map(b -> {
+                if (b == null)
+                    return null;
+                DurablyAppliedTo result = new DurablyAppliedTo(b);
+                if (!include.test(result))
+                    return null;
+                return result;
+            }, DurablyAppliedTo[]::new);
+        }
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordCommandStores.java

@@ -17,11 +17,18 @@
  */
 package org.apache.cassandra.service.accord;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.Journal;
@@ -32,12 +39,20 @@ import accord.local.NodeCommandStoreService;
 import accord.local.SequentialAsyncExecutor;
 import accord.local.ShardDistributor;
 import accord.utils.RandomSource;
+import accord.utils.Reduce;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
 
 import org.apache.cassandra.cache.CacheSize;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.config.AccordSpec.QueueShardModel;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.journal.Descriptor;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordCommandStore.DurablyAppliedTo;
 import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory;
 
 import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_PER_SHARD;
@@ -46,10 +61,13 @@ import static org.apache.cassandra.config.DatabaseDescriptor.getAccordQueueSubmi
 import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITHOUT_LOCK;
 import static org.apache.cassandra.service.accord.AccordExecutor.Mode.RUN_WITH_LOCK;
 import static org.apache.cassandra.service.accord.AccordExecutor.constant;
+import static org.apache.cassandra.service.accord.journal.ReplayMarkers.saveDirectory;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class AccordCommandStores extends CommandStores implements CacheSize, Shutdownable
 {
+    private static final Logger logger = LoggerFactory.getLogger(AccordCommandStores.class);
+
     private final AccordExecutor[] executors;
     private final int mask;
 
@@ -216,14 +234,49 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu
         return Stream.of(executors).allMatch(AccordExecutor::isTerminated);
     }
 
-    @Override
-    public synchronized void shutdown()
+    public Set<TableId> shutdownStores()
+    {
+        markShuttingDown();
+        Set<TableId> tableIds = new HashSet<>();
+        List<AsyncResult<Void>> async = new ArrayList<>();
+        for (ShardHolder shard : current())
+        {
+            AccordCommandStore commandStore = (AccordCommandStore) shard.store;
+            tableIds.add(commandStore.tableId());
+            async.add(commandStore.shutdownAsync());
+        }
+        if (!async.isEmpty())
+            AccordService.getBlocking(AsyncResults.reduce(async, Reduce.toNull()));
+        return tableIds;
+    }
+
+    public boolean awaitStoreTermination(long deadlineNanos)
+    {
+        for (ShardHolder shard : current())
+        {
+            AccordCommandStore commandStore = (AccordCommandStore) shard.store;
+            if (!commandStore.awaitTerminationUntil(deadlineNanos))
+            {
+                logger.warn("{} timeout awaiting durability: {}", commandStore, DurablyAppliedTo.summarise(commandStore.unsafeGetRedundantBefore()));
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public void shutdownExecutors()
     {
-        super.shutdown();
         for (AccordExecutor executor : executors)
             executor.shutdown();
     }
 
+    @Override
+    public void shutdown()
+    {
+        shutdownStores();
+        shutdownExecutors();
+    }
+
     @Override
     public Object shutdownNow()
     {
@@ -243,4 +296,27 @@ public class AccordCommandStores extends CommandStores implements CacheSize, Shu
         }
         return true;
     }
+
+    public AsyncChain<Boolean> saveState(Descriptor descriptor)
+    {
+        saveDirectory().createDirectoriesIfNotExists();
+        List<AsyncChain<Boolean>> chains = new ArrayList<>();
+        for (ShardHolder shard : current())
+        {
+            AccordCommandStore commandStore = (AccordCommandStore)shard.store;
+            chains.add(commandStore.saveState(descriptor));
+        }
+        return AsyncChains.reduce(chains, Boolean::logicalAnd, true);
+    }
+
+    public AsyncChain<List<Map.Entry<Integer, Long>>> restoreState()
+    {
+        List<AsyncChain<Map.Entry<Integer, Long>>> chains = new ArrayList<>();
+        for (ShardHolder shard : current())
+        {
+            AccordCommandStore commandStore = (AccordCommandStore)shard.store;
+            chains.add(commandStore.restoreState());
+        }
+        return AsyncChains.allOf(chains);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordDataStore.java

@@ -18,6 +18,11 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -26,15 +31,36 @@ import accord.local.CommandStore;
 import accord.local.Node;
 import accord.local.RedundantBefore;
 import accord.local.SafeCommandStore;
-import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.SyncPoint;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import accord.utils.Reduce;
 import accord.utils.UnhandledEnum;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
 
+import org.apache.cassandra.concurrent.ScheduledExecutors;
 import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.UnknownTableException;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.repair.RepairCoordinator;
+import org.apache.cassandra.repair.RepairParallelism;
+import org.apache.cassandra.repair.messages.RepairOption;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.tcm.ClusterMetadata;
+
+import static accord.local.durability.DurabilityService.SyncLocal.NoLocal;
+import static accord.local.durability.DurabilityService.SyncRemote.Quorum;
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.utils.Invariants.illegalArgument;
 
 public class AccordDataStore implements DataStore
 {
@@ -45,75 +71,154 @@ public class AccordDataStore implements DataStore
      * Ensures data for the intersecting ranges is flushed to sstable before calling back with reportOnSuccess.
      * This is used to gate journal cleanup, since we skip the CommitLog for applying to the data table.
      */
-    public void ensureDurable(CommandStore commandStore, Ranges ranges, RedundantBefore reportOnSuccess)
+    @Override
+    public void ensureDurable(CommandStore commandStore, Ranges ranges, RedundantBefore reportOnSuccess, int flags)
     {
-        if (commandStore.node().isReplaying())
+        if (commandStore.node().isReplaying() || ranges.isEmpty())
             return;
 
-        logger.debug("{} awaiting local data durability of {}", commandStore, ranges);
-        ColumnFamilyStore prev = null;
-        for (Range range : ranges)
-        {
-            ColumnFamilyStore cfs;
-            if (prev != null && prev.metadata().id.equals(range.prefix())) cfs = prev;
-            else cfs = Schema.instance.getColumnFamilyStoreInstance((TableId) range.prefix());
-            if (cfs == null)
-            {
-                // TODO (expected): should we record this as durable?
-                continue;
-            }
-
-            while (true)
-            {
-                Memtable memtable = cfs.getCurrentMemtable();
-                // If RX came when after a quiet period or if it raced with a previous memtable flush
-                if (memtable.isClean())
-                {
-                    AccordDurableOnFlush.notify(cfs.metadata(), commandStore, reportOnSuccess);
-                    break;
-                }
-
-                AccordDurableOnFlush onFlush = memtable.ensureFlushListener(FlushListenerKey.KEY, AccordDurableOnFlush::new);
-                if (onFlush != null && onFlush.add(commandStore.id(), reportOnSuccess))
-                    break;
-
-                if (cfs == prev)
-                {
-                    // we must already have a successful notify, so just propagate
-                    AccordDurableOnFlush.notify(cfs.metadata(), commandStore, reportOnSuccess);
-                    break;
-                }
-            }
-
-            prev = cfs;
-        }
+        logger.debug("{} awaiting local data durability for {}", commandStore, ranges);
+        ensureDurableInternal(commandStore, reportOnSuccess, flags);
     }
 
     @Override
-    public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback, FetchKind kind)
+    public void ensureDurable(CommandStore commandStore, RedundantBefore reportOnSuccess, int flags)
     {
-        switch (kind)
-        {
-            default: throw new UnhandledEnum(kind);
-            case Image:
-            {
-                AccordFetchCoordinator coordinator;
-                try
-                {
-                    coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore());
-                }
-                catch (Throwable t)
-                {
-                    return new FetchResult.Failure(t);
-                }
+        logger.debug("{} awaiting full local data durability", commandStore);
+        ensureDurableInternal(commandStore, reportOnSuccess, flags);
+    }
 
-                coordinator.start();
-                return coordinator.result();
-            }
-            case Sync:
+    private void ensureDurableInternal(CommandStore commandStore, RedundantBefore redundantBefore, int flags)
+    {
+        ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(((AccordCommandStore)commandStore).tableId());
+        AccordDurableOnFlush.notifyOnDurable(cfs, commandStore, ReportDurable.of(redundantBefore, flags));
+    }
+
+    public FetchResult image(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback)
+    {
+        AccordFetchCoordinator coordinator;
+        try
+        {
+            coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore());
+        }
+        catch (Throwable t)
+        {
+            return new FetchResult.Failure(t);
+        }
+
+        coordinator.start();
+        return coordinator.result();
+    }
+
+    public FetchResult sync(Node node, SafeCommandStore safeStore, Map<TxnId, Ranges> rangesById, FetchRanges callback)
+    {
+        TableId tableId = ((AccordCommandStore)safeStore.commandStore()).tableId();
+        Invariants.require(rangesById.values().stream().flatMap(Ranges::stream).allMatch(r -> tableId.equals(r.prefix())));
+        Ranges ranges = rangesById.values().stream().reduce(Ranges::with).get();
+
+        ClusterMetadata cm = ClusterMetadata.current();
+        TableMetadata tableMetadata = cm.schema.getTableMetadata(tableId);
+        if (tableMetadata == null)
+        {
+            Throwable fail = new UnknownTableException("Could not find tableId " + tableId + " in ClusterMetadata", tableId);
+            callback.fail(ranges, fail);
+            return new FetchResult.Failure(fail);
+        }
+
+        class SyncResult extends AsyncResults.SettableResult<Ranges> implements FetchResult
+        {
+            @Override
+            public void abort(Ranges ranges)
             {
-                throw new UnsupportedOperationException();
+                throw new UnsupportedOperationException("Can not abort sync task");
             }
         }
+
+        // TODO (expected): add some automatic slicing of ranges and retry/back-off logic; but for now,
+        //  since this is done at the command store level, and this is already a slice of a node, this should be fine
+        SyncResult syncResult = new SyncResult();
+
+        logger.info("Requesting quorum durability before initiating repair");
+        List<AsyncResult<Void>> syncs = new ArrayList<>();
+        for (Map.Entry<TxnId, Ranges> e : rangesById.entrySet())
+            syncs.add(node.durability().sync("Sync Data Store", ExclusiveSyncPoint, e.getKey(), e.getValue(), NoLocal, Quorum, 1L, TimeUnit.HOURS));
+
+        AsyncResults.reduce(syncs, Reduce.toNull()).invoke((success, fail) -> {
+            if (fail != null)
+            {
+                logger.error("{} failed to achieve quorum durability before repair for rebootstrap of {}", safeStore.commandStore(), ranges);
+                syncResult.tryFailure(fail);
+                return;
+            }
+
+            RepairCoordinator coord = StorageService.instance.newRepairCoordinator(tableMetadata.keyspace, options(tableMetadata, ranges));
+            coord.addProgressListener((tag, event) -> {
+                switch (event.getType())
+                {
+                    default: throw new UnhandledEnum(event.getType());
+                    case START:
+                        callback.starting(ranges);
+                        break;
+                    case PROGRESS:
+                    case COMPLETE:
+                    case NOTIFICATION:
+                        break;
+                    case ABORT:
+                    case ERROR:
+                        IllegalStateException ex = new IllegalStateException(String.format("Repair failed: %s", event));
+                        callback.fail(ranges, ex);
+                        syncResult.tryFailure(ex);
+                        break;
+                    case SUCCESS:
+                        callback.fetched(ranges);
+                        syncResult.trySuccess(null);
+                        break;
+                }
+            });
+
+            ScheduledExecutors.optionalTasks.submit(coord).addCallback((s, f) -> {
+                if (f != null)
+                    syncResult.tryFailure(f);
+            });
+        });
+
+        return syncResult;
+    }
+
+    private static RepairOption options(TableMetadata tableMetadata, Ranges accordRanges)
+    {
+        List<Range<Token>> ranges = new ArrayList<>();
+        RangesAtEndpoint localRanges = StorageService.instance.getLocalReplicas(tableMetadata.keyspace);
+        // repair validation requires that we separate by local range, even if they are adjacent;
+        // unsure if this is important, so just splitting into ranges that are wholly contained by local ranges
+        accordRanges.forEach(accordRange -> {
+            Range<Token> range = ((TokenRange)accordRange).toKeyspaceRange();
+            localRanges.ranges().forEach(localRange -> {
+                if (localRange.contains(range)) ranges.add(range);
+                else if (localRange.intersects(range)) ranges.addAll(range.intersectionWith(localRange));
+            });
+        });
+
+        Ranges matchedWithLocal = Ranges.of(ranges.stream().map(r -> TokenRange.fromKeyspaceRange(tableMetadata.id, r)).toArray(TokenRange[]::new)).mergeTouching();
+        if (!matchedWithLocal.containsAll(accordRanges))
+            throw illegalArgument("Local ranges %s do not fully cover requested accord ranges %s (overlap: %s)", localRanges, accordRanges, matchedWithLocal);
+
+        return new RepairOption(RepairParallelism.PARALLEL, // parallelism
+                false,                      // primaryRange
+                false,                      // incremental
+                false,                      // trace
+                5,                          // jobThreads
+                ranges,                     // ranges
+                true,                       // pullRepair
+                true,                       // forceRepair
+                PreviewKind.NONE,           // previewKind
+                false,                      // optimiseStreams
+                true,                       // ignoreUnreplicatedKeyspaces
+                true,                       // repairData
+                false,                      // repairPaxos
+                true,                       // dontPurgeTombstones
+                false,                      // repairAccord
+                false
+        );
     }
 }

src/java/org/apache/cassandra/service/accord/AccordDurableOnFlush.java

@@ -19,7 +19,7 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.Map;
-import java.util.function.Consumer;
+import java.util.function.BiConsumer;
 
 import org.agrona.collections.Int2ObjectHashMap;
 import org.slf4j.Logger;
@@ -27,51 +27,164 @@ import org.slf4j.LoggerFactory;
 
 import accord.local.CommandStore;
 import accord.local.CommandStores;
-import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
 
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 
-class AccordDurableOnFlush implements Consumer<TableMetadata>
+class AccordDurableOnFlush implements BiConsumer<Long, TableMetadata>
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordDurableOnFlush.class);
 
-    private Int2ObjectHashMap<RedundantBefore> commandStores = new Int2ObjectHashMap<>();
+    public static class ReportDurable
+    {
+        public static final int COMMAND_STORE_FLUSH = 1;
+        public static final int DATA_STORE_FLUSH = 2;
+
+        public final RedundantBefore redundantBefore;
+        final int flags;
+
+        private ReportDurable(RedundantBefore redundantBefore, int flags)
+        {
+            this.redundantBefore = redundantBefore;
+            this.flags = flags;
+        }
+
+        public boolean isDataStoreFlush()
+        {
+            return isDataStoreFlush(flags);
+        }
+
+        public static boolean isDataStoreFlush(int flags)
+        {
+            return 0 != (flags & DATA_STORE_FLUSH);
+        }
+
+        public boolean isCommandStoreFlush()
+        {
+            return isCommandStoreFlush(flags);
+        }
+
+        public static boolean isCommandStoreFlush(int flags)
+        {
+            return 0 != (flags & COMMAND_STORE_FLUSH);
+        }
+
+        public static ReportDurable of(RedundantBefore redundantBefore)
+        {
+            return of(redundantBefore, 0);
+        }
+
+        public static ReportDurable of(RedundantBefore redundantBefore, int flags)
+        {
+            return new ReportDurable(redundantBefore, flags);
+        }
+
+        public static ReportDurable commandStoreFlush()
+        {
+            return new ReportDurable(RedundantBefore.EMPTY, COMMAND_STORE_FLUSH);
+        }
+
+        static ReportDurable merge(ReportDurable a, ReportDurable b)
+        {
+            return new ReportDurable(RedundantBefore.merge(a.redundantBefore, b.redundantBefore), a.flags | b.flags);
+        }
+
+        @Override
+        public String toString()
+        {
+            return redundantBefore.toString();
+        }
+    }
+
+    private Int2ObjectHashMap<ReportDurable> commandStores = new Int2ObjectHashMap<>();
 
     AccordDurableOnFlush()
     {
     }
 
-    synchronized boolean add(int commandStoreId, RedundantBefore reportOnFlush)
+    synchronized boolean add(int commandStoreId, ReportDurable reportOnFlush)
     {
         if (commandStores == null)
             return false;
-        commandStores.merge(commandStoreId, reportOnFlush, RedundantBefore::merge);
+        commandStores.merge(commandStoreId, reportOnFlush, ReportDurable::merge);
         return true;
     }
 
     @Override
-    public void accept(TableMetadata metadata)
+    public void accept(Long memtableId, TableMetadata metadata)
     {
-        Int2ObjectHashMap<RedundantBefore> notify;
+        Int2ObjectHashMap<ReportDurable> notify;
         synchronized (this)
         {
             notify = commandStores;
             commandStores = null;
         }
         CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
-        for (Map.Entry<Integer, RedundantBefore> e : notify.entrySet())
+        for (Map.Entry<Integer, ReportDurable> e : notify.entrySet())
         {
-            RedundantBefore durable = e.getValue();
-            notify(metadata, commandStores.forId(e.getKey()), durable);
+            ReportDurable durable = e.getValue();
+            notifyInOrder(memtableId, metadata, commandStores.forId(e.getKey()), durable);
         }
     }
 
-    static void notify(TableMetadata metadata, CommandStore commandStore, RedundantBefore report)
+    public static void notifyOnDurable(ColumnFamilyStore cfs, CommandStore commandStore, ReportDurable onDurable)
     {
-        logger.debug("Reporting flush of {}/{}; reporting {} to {}", metadata.id, metadata, report, commandStore);
-        commandStore.execute((PreLoadContext.Empty) () -> "Report Durable", safeStore -> {
-            safeStore.upsertRedundantBefore(report);
-        });
+        if (cfs == null)
+        {
+            // TODO (required): is this correct? Revisit when we improve DROP TABLE
+            notifyNow(commandStore, onDurable);
+            return;
+        }
+        View view = cfs.getTracker().getView();
+        for (int i = view.liveMemtables.size() - 1; i >= 0 ; --i)
+        {
+            Memtable candidate = view.liveMemtables.get(i);
+            if (candidate.isClean())
+                continue;
+
+            AccordDurableOnFlush onFlush = candidate.ensureFlushListener(AccordDataStore.FlushListenerKey.KEY, AccordDurableOnFlush::new);
+            if (onFlush != null && onFlush.add(commandStore.id(), onDurable))
+                return;
+        }
+
+        for (int i = view.flushingMemtables.size() - 1; i >= 0 ; --i)
+        {
+            Memtable candidate = view.flushingMemtables.get(i);
+            AccordDurableOnFlush onFlush = candidate.ensureFlushListener(AccordDataStore.FlushListenerKey.KEY, AccordDurableOnFlush::new);
+            if (onFlush != null && onFlush.add(commandStore.id(), onDurable))
+                return;
+        }
+
+        notifyNow(commandStore, onDurable);
+    }
+
+    static void notifyInOrder(long memtableId, TableMetadata metadata, CommandStore commandStore, ReportDurable report)
+    {
+        ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
+        if (cfs == null)
+        {
+            notifyNow(commandStore, report);
+            return;
+        }
+        View view = cfs.getTracker().getView();
+        boolean notifyNow = true;
+        for (Memtable memtable : view.liveMemtables)
+            notifyNow &= memtable.getMemtableId() > memtableId;
+        for (Memtable memtable : view.flushingMemtables)
+            notifyNow &= memtable.getMemtableId() > memtableId;
+        if (notifyNow) notifyNow(commandStore, report);
+        else cfs.waitForPriorFlushes().addListener(() -> notifyNow(commandStore, report));
+    }
+
+    static void notifyNow(CommandStore commandStore, ReportDurable report)
+    {
+        logger.debug("{} reporting flush with {}", commandStore, report);
+        commandStore.execute((AccordExecutor.Unstoppable) () -> "Report Durable", safeStore -> {
+            safeStore.reportDurable(report.redundantBefore, report.flags);
+        }, commandStore.agent());
     }
 }

src/java/org/apache/cassandra/service/accord/AccordExecutor.java

@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.LockSupport;
@@ -86,6 +87,8 @@ import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Condition;
 import org.apache.cassandra.utils.concurrent.Future;
 
+import io.netty.util.concurrent.FastThreadLocal;
+
 import static accord.utils.Invariants.createIllegalState;
 import static org.apache.cassandra.service.accord.AccordCache.CommandAdapter.COMMAND_ADAPTER;
 import static org.apache.cassandra.service.accord.AccordCache.CommandsForKeyAdapter.CFK_ADAPTER;
@@ -116,13 +119,11 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
     // WARNING: this is a shared object, so close is NOT idempotent
     public static final class ExclusiveGlobalCaches extends GlobalCaches implements AutoCloseable
     {
-        final Lock lock;
         final AccordExecutor executor;
 
         public ExclusiveGlobalCaches(AccordExecutor executor, AccordCache global, AccordCache.Type<TxnId, Command, AccordSafeCommand> commands, AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKey)
         {
             super(global, commands, commandsForKey);
-            this.lock = executor.lock;
             this.executor = executor;
         }
 
@@ -130,8 +131,8 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         public void close()
         {
             executor.beforeUnlock();
-            global.tryShrinkOrEvict(lock);
-            lock.unlock();
+            global.tryShrinkOrEvict(executor.lock);
+            executor.unlock();
         }
     }
 
@@ -149,7 +150,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         }
     }
 
-    final Lock lock;
+    private final Lock lock;
     final Agent agent;
     final int executorId;
     private final AccordCache cache;
@@ -245,9 +246,57 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
 
     public ExclusiveGlobalCaches lockCaches()
     {
+        lock();
+        return caches;
+    }
+
+    private static final FastThreadLocal<Lock> paranoidPriorityInversionCheck = new FastThreadLocal<>();
+
+    final Lock unsafeLock()
+    {
+        return lock;
+    }
+
+    final void lock()
+    {
+        if (Invariants.isParanoid())
+        {
+            Lock locked = paranoidPriorityInversionCheck.getAndSet(lock);
+            Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress");
+        }
         //noinspection LockAcquiredButNotSafelyReleased
         lock.lock();
-        return caches;
+    }
+
+    final void unlock()
+    {
+        lock.unlock();
+        paranoidPriorityInversionCheck.set(null);
+    }
+
+    final boolean tryLock()
+    {
+        boolean result = lock.tryLock();
+        if (Invariants.isParanoid())
+        {
+            if (result)
+            {
+                Lock locked = paranoidPriorityInversionCheck.getAndSet(lock);
+                if (locked != null && locked != lock)
+                {
+                    lock.unlock();
+                    paranoidPriorityInversionCheck.set(locked);
+                    Invariants.require(false, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress");
+                    return false;
+                }
+            }
+            else
+            {
+                Lock locked = paranoidPriorityInversionCheck.get();
+                Invariants.require(locked == null || locked == lock, "Tried to take multiple AccordExecutor locks on same thread - this is dangerous for progress");
+            }
+        }
+        return result;
     }
 
     public AccordCache cacheExclusive()
@@ -289,7 +338,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
     public void waitForQuiescence()
     {
         Condition condition;
-        lock.lock();
+        lock();
         try
         {
             if (tasks == 0 && runningThreads == 0)
@@ -302,7 +351,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         }
         finally
         {
-            lock.unlock();
+            unlock();
         }
         condition.awaitThrowUncheckedOnInterrupt();
     }
@@ -324,7 +373,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
 
     public void afterSubmittedAndConsequences(Runnable run)
     {
-        lock.lock();
+        lock();
         try
         {
             if (tasks == 0 && runningThreads == 0)
@@ -343,7 +392,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         }
         finally
         {
-            lock.unlock();
+            unlock();
         }
     }
 
@@ -599,7 +648,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         enqueueLoadsExclusive();
     }
 
-    void submitExclusive(Runnable runnable)
+    public void submitExclusive(Runnable runnable)
     {
         submitPlainExclusive(new PlainRunnable(null, runnable));
     }
@@ -710,7 +759,6 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
                 finally { completeTaskExclusive(task); }
                 break;
 
-            case FAILING:
             case RUNNING:
             case PERSISTING:
             case FINISHED:
@@ -814,7 +862,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
 
     public void executeDirectlyWithLock(Runnable command)
     {
-        lock.lock();
+        lock();
         try
         {
             command.run();
@@ -822,7 +870,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         finally
         {
             beforeUnlock();
-            lock.unlock();
+            unlock();
         }
     }
 
@@ -900,6 +948,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
     public static abstract class Task extends IntrusivePriorityHeap.Node
     {
         int queuePosition;
+        Thread assigned;
 
         protected Task()
         {
@@ -941,6 +990,16 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         abstract void submitExclusive(AccordExecutor owner);
     }
 
+    // run the task even on a stopped commandStore
+    public interface Unstoppable extends PreLoadContext.Empty
+    {
+    }
+
+    // run the task even on a terminated commandStore
+    public interface Unterminatable extends Unstoppable
+    {
+    }
+
     static class SequentialQueueTask extends Task
     {
         private final SequentialExecutor queue;
@@ -989,8 +1048,11 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         final int commandStoreId;
         final SequentialQueueTask selfTask;
         private Task task;
+        private Thread assigned;
         private volatile Thread owner, waiting;
-        private boolean running;
+        private boolean stopped;
+        private volatile boolean visibleStopped;
+        private boolean terminated;
 
         SequentialExecutor()
         {
@@ -1007,21 +1069,46 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         void preRunTask()
         {
             Invariants.require(task != null);
+            assigned = Thread.currentThread();
             task.preRunExclusive();
-            running = true;
         }
 
         void runTask()
         {
-            Thread self = Thread.currentThread();
-            while (!ownerUpdater.compareAndSet(this, null, self))
+            outer: while (!ownerUpdater.compareAndSet(this, null, assigned))
             {
-                waiting = self;
-                while (owner != null)
+                waiting = assigned;
+                while (true)
+                {
+                    Thread owner = this.owner;
+                    if (owner == assigned) break outer;
+                    if (owner == null) continue outer;
                     LockSupport.park();
-                waiting = null;
+                }
             }
-            task.runInternal();
+            waiting = null;
+
+            try
+            {
+                if (stopped && reject(task))
+                    task.fail(new RejectedExecutionException(commandStoreId + " is terminated. Cannot execute " + ((AccordTask<?>) task).preLoadContext()));
+                else
+                    task.runInternal();
+            }
+            finally
+            {
+                owner = null;
+            }
+        }
+
+        private boolean reject(Task task)
+        {
+            if (!(task instanceof AccordTask<?>))
+                return true;
+
+            PreLoadContext context = ((AccordTask<?>) task).preLoadContext();
+
+            return !(terminated ? (context instanceof Unterminatable) : (context instanceof Unstoppable));
         }
 
         void failTask(Throwable t)
@@ -1034,9 +1121,10 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
             try { task.cleanupExclusive(); }
             finally
             {
-                owner = null;
-                running = false;
+                assigned = null;
                 task = super.poll();
+
+                // it should only be possible for this method to be invoked once we're on the running queue
                 AccordExecutor.this.running.remove(selfTask);
                 if (task != null)
                 {
@@ -1052,12 +1140,12 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         {
             if (task != null)
             {
-                Invariants.require(running || waitingToRun.contains(selfTask));
+                Invariants.require(assigned != null || waitingToRun.contains(selfTask));
                 super.append(newTask);
             }
             else
             {
-                Invariants.require(!running && isEmpty());
+                Invariants.require(assigned == null && isEmpty());
                 task = newTask;
                 selfTask.queuePosition = newTask.queuePosition;
                 waitingToRun.append(selfTask);
@@ -1067,37 +1155,47 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         @Override
         protected void remove(Task remove)
         {
-            Invariants.require(remove != null);
-            if (remove != task)
-            {
-                super.remove(remove);
-            }
-            else if (!running)
-            {
-                // cannot overwrite task while it is being executed - this cannot happen for AccordTask
-                // but can for other tasks that don't track their own state
+            if (remove == task) removeCurrentTask(remove);
+            else super.remove(remove);
+        }
 
-                task = super.poll();
-                if (waitingToRun.contains(selfTask))
-                {
-                    if (task == null) waitingToRun.remove(selfTask);
-                    else
-                    {
-                        selfTask.queuePosition = task.queuePosition;
-                        waitingToRun.update(selfTask);
-                    }
-                }
+        @Override
+        protected boolean removeIfContains(Task remove)
+        {
+            if (remove == task) return removeCurrentTask(remove);
+            else return super.removeIfContains(remove);
+        }
+
+        private boolean removeCurrentTask(Node remove)
+        {
+            if (assigned != null)
+                return false;
+
+            Invariants.require(remove == task);
+            // cannot overwrite task while it is being executed - this cannot happen for AccordTask
+            // but can for other tasks that don't track their own state
+
+            task = super.poll();
+            if (waitingToRun.contains(selfTask))
+            {
+                if (task == null) waitingToRun.remove(selfTask);
                 else
                 {
-                    Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove);
-                    if (task != null)
-                    {
-                        selfTask.queuePosition = task.queuePosition;
-                        waitingToRun.append(selfTask);
-                    }
+                    selfTask.queuePosition = task.queuePosition;
+                    waitingToRun.update(selfTask);
                 }
             }
-            Invariants.require(task == null || running || waitingToRun.contains(selfTask));
+            else
+            {
+                Invariants.expect(false, "%s should have been queued to run as it had the task %s pending, that has now been cancelled", this, remove);
+                if (task != null)
+                {
+                    selfTask.queuePosition = task.queuePosition;
+                    waitingToRun.append(selfTask);
+                }
+            }
+            Invariants.require(task == null || waitingToRun.contains(selfTask));
+            return true;
         }
 
         public boolean inExecutor()
@@ -1105,6 +1203,24 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
             return owner == Thread.currentThread();
         }
 
+        public boolean stopped()
+        {
+            return visibleStopped;
+        }
+
+        void stop()
+        {
+            Invariants.require(inExecutor());
+            this.stopped = true;
+            this.visibleStopped = true;
+        }
+
+        void terminate()
+        {
+            Invariants.require(inExecutor());
+            this.visibleStopped = this.terminated = this.stopped = true;
+        }
+
         @Override
         protected Task poll()
         {
@@ -1120,7 +1236,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         @Override
         protected boolean contains(Task contains)
         {
-            return super.contains(contains) || (task == contains && !running);
+            return super.contains(contains) || (task == contains && assigned == null);
         }
 
         @Override
@@ -1205,7 +1321,10 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
                 {
                     this.owner = null;
                     if (waiting != null)
+                    {
                         LockSupport.unpark(waiting);
+                        ownerUpdater.compareAndSet(this, null, waiting);
+                    }
                 }
             }
             return true;
@@ -1259,6 +1378,12 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
             super.remove(remove);
         }
 
+        @Override
+        protected boolean removeIfContains(T node)
+        {
+            return super.removeIfContains(node);
+        }
+
         protected boolean contains(T contains)
         {
             return super.contains(contains);
@@ -1607,8 +1732,8 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
             }
             catch (Throwable t)
             {
+                // shouldn't throw exceptions
                 agent.onException(t);
-                return;
             }
         }
 
@@ -1741,7 +1866,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
     public List<TaskInfo> taskSnapshot()
     {
         List<TaskInfo> result = new ArrayList<>();
-        lock.lock();
+        lock();
         try
         {
             addToSnapshot(result, waitingToLoad, TaskInfo.Status.WAITING_TO_LOAD, TaskInfo.Status.WAITING_TO_LOAD);
@@ -1753,7 +1878,7 @@ public abstract class AccordExecutor implements CacheSize, LoadExecutor<AccordTa
         }
         finally
         {
-            lock.unlock();
+            unlock();
         }
         result.sort(TaskInfo::compareTo);
         return result;

src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractLockLoop.java

@@ -84,14 +84,14 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
         if (tasks > 0 || !submitted.isEmpty() || runningThreads > 0)
             return true;
 
-        lock.lock();
+        lock();
         try
         {
             return tasks > 0 || !submitted.isEmpty() || runningThreads > 0;
         }
         finally
         {
-            lock.unlock();
+            unlock();
         }
     }
 
@@ -157,7 +157,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
                 Task task;
                 while (true)
                 {
-                    lock.lock();
+                    lock();
                     try
                     {
                         resumeExclusive();
@@ -210,7 +210,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
                     }
                     finally
                     {
-                        lock.unlock();
+                        unlock();
                     }
                 }
             }
@@ -227,7 +227,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
                 Task task = null;
                 while (true)
                 {
-                    lock.lock();
+                    lock();
                     try
                     {
                         if (task != null)
@@ -288,7 +288,7 @@ abstract class AccordExecutorAbstractLockLoop extends AccordExecutor
                     }
                     finally
                     {
-                        lock.unlock();
+                        unlock();
                     }
 
                     try

src/java/org/apache/cassandra/service/accord/AccordExecutorAbstractSemiSyncSubmit.java

@@ -35,7 +35,7 @@ abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstra
 
     <P1s, P1a, P2, P3, P4> void submitExternal(QuintConsumer<AccordExecutor, P1s, P2, P3, P4> sync, QuadFunction<P1a, P2, P3, P4, Submittable> async, P1s p1s, P1a p1a, P2 p2, P3 p3, P4 p4)
     {
-        if (!lock.tryLock())
+        if (!tryLock())
         {
             submitted.push(async.apply(p1a, p2, p3, p4));
             notifyWork();
@@ -48,7 +48,7 @@ abstract class AccordExecutorAbstractSemiSyncSubmit extends AccordExecutorAbstra
         }
         finally
         {
-            lock.unlock();
+            unlock();
         }
     }
 }

src/java/org/apache/cassandra/service/accord/AccordJournalTable.java

@@ -1,646 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.accord;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
-
-import javax.annotation.CheckForNull;
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.AbstractIterator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.local.MaxDecidedRX;
-import accord.primitives.Timestamp;
-import accord.primitives.TxnId;
-import accord.utils.Invariants;
-import accord.utils.UncheckedInterruptedException;
-
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.ColumnIdentifier;
-import org.apache.cassandra.cql3.Operator;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.ColumnFamilyStore.RefViewFragment;
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.EmptyIterators;
-import org.apache.cassandra.db.PartitionRangeReadCommand;
-import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.Slices;
-import org.apache.cassandra.db.StorageHook;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.filter.DataLimits;
-import org.apache.cassandra.db.filter.RowFilter;
-import org.apache.cassandra.db.lifecycle.SSTableSet;
-import org.apache.cassandra.db.lifecycle.View;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.db.rows.UnfilteredRowIterators;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.accord.OrderedRouteSerializer;
-import org.apache.cassandra.index.accord.RouteJournalIndex;
-import org.apache.cassandra.io.FSReadError;
-import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.journal.Journal;
-import org.apache.cassandra.journal.KeySupport;
-import org.apache.cassandra.journal.RecordConsumer;
-import org.apache.cassandra.journal.Segment;
-import org.apache.cassandra.journal.Segments;
-import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.service.RetryStrategy;
-import org.apache.cassandra.service.accord.AccordKeyspace.JournalColumns;
-import org.apache.cassandra.service.accord.api.TokenKey;
-import org.apache.cassandra.service.accord.serializers.CommandSerializers;
-import org.apache.cassandra.service.accord.serializers.Version;
-import org.apache.cassandra.utils.Closeable;
-import org.apache.cassandra.utils.CloseableIterator;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.MergeIterator;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-
-import static org.apache.cassandra.io.sstable.SSTableReadsListener.NOOP_LISTENER;
-import static org.apache.cassandra.service.accord.AccordKeyspace.JournalColumns.getJournalKey;
-
-public class AccordJournalTable<K extends JournalKey, V> implements JournalRangeSearcher.Supplier
-{
-    private static final Logger logger = LoggerFactory.getLogger(AccordJournalTable.class);
-
-    private final Journal<K, V> journal;
-    private final ColumnFamilyStore cfs;
-
-    private final ColumnMetadata recordColumn;
-    private final ColumnMetadata versionColumn;
-
-    private final KeySupport<K> keySupport;
-    /**
-     * Access to this field should only ever be handled by {@link #safeNotify(Consumer)}.  There is an assumption that
-     * an error in the index should not cause the journal to crash, so {@link #safeNotify(Consumer)} exists to make sure
-     * this property holds true.
-     */
-    @Nullable
-    private final JournalSegmentRangeSearcher<Object> index;
-    private final Version accordJournalVersion;
-
-    public AccordJournalTable(Journal<K, V> journal, KeySupport<K> keySupport, ColumnFamilyStore cfs, Version accordJournalVersion)
-    {
-        this.journal = journal;
-        this.cfs = cfs;
-        this.recordColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("record", false));
-        this.versionColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("user_version", false));
-        this.keySupport = keySupport;
-        this.accordJournalVersion = accordJournalVersion;
-
-        this.index = cfs.indexManager.getIndexByName(AccordKeyspace.JOURNAL_INDEX_NAME) != null
-                     ? new JournalSegmentRangeSearcher<>()
-                     : null;
-    }
-
-    boolean shouldIndex(JournalKey key)
-    {
-        if (index == null) return false;
-        return RouteJournalIndex.allowed(key);
-    }
-
-    void safeNotify(Consumer<JournalSegmentRangeSearcher<Object>> fn)
-    {
-        if (index == null)
-            return;
-        try
-        {
-            fn.accept(index);
-        }
-        catch (Throwable t)
-        {
-            JVMStabilityInspector.inspectThrowable(t);
-            logger.warn("Failure updating index", t);
-        }
-    }
-
-    public void forceCompaction()
-    {
-        cfs.forceMajorCompaction();
-    }
-
-    @Override
-    public JournalRangeSearcher rangeSearcher()
-    {
-        if (index == null)
-            return JournalRangeSearcher.NoopJournalRangeSearcher.instance;
-        return new JournalTableRangeSearcher();
-    }
-
-    public void start()
-    {
-        if (index == null) return;
-        Index tableIndex = cfs.indexManager.getIndexByName(AccordKeyspace.JOURNAL_INDEX_NAME);
-        RetryStrategy retry = DatabaseDescriptor.getAccord().retry_journal_index_ready.retry();
-        for (int i = 0; !cfs.indexManager.isIndexQueryable(tableIndex); i++)
-        {
-            logger.debug("Journal index {} is not ready wait... waiting", AccordKeyspace.JOURNAL_INDEX_NAME);
-            maybeWait(retry, i);
-        }
-    }
-
-    /**
-     * This method is here to make it easier for org.apache.cassandra.distributed.test.accord.journal.JournalAccessRouteIndexOnStartupRaceTest
-     * to check when we need to do waiting
-     */
-    @VisibleForTesting
-    private static void maybeWait(RetryStrategy retry, int i)
-    {
-        long waitTime = retry.computeWait(i, TimeUnit.MICROSECONDS);
-        if (waitTime == -1)
-            throw new IllegalStateException("Gave up waiting on journal index to be ready");
-        try
-        {
-            TimeUnit.MICROSECONDS.sleep(waitTime);
-        }
-        catch (InterruptedException e)
-        {
-            throw new UncheckedInterruptedException(e);
-        }
-    }
-
-    public interface Reader
-    {
-        void read(DataInputPlus input, Version userVersion) throws IOException;
-    }
-
-    static class RecordConsumerAdapter<K> implements RecordConsumer<K>
-    {
-        protected final Reader reader;
-
-        RecordConsumerAdapter(Reader reader)
-        {
-            this.reader = reader;
-        }
-
-        private long prevSegment = Long.MAX_VALUE;
-        private long prevPosition = Long.MAX_VALUE;
-
-        @Override
-        public void accept(long segment, int position, K key, ByteBuffer buffer, int userVersion)
-        {
-            Invariants.require(segment <= prevSegment,
-                               "Records should always be iterated over in a reverse order, but segment %d was seen after %d while reading %s", segment, prevSegment, key);
-            Invariants.require(segment != prevSegment || position < prevPosition,
-                               "Records should always be iterated over in a reverse order, but position %d was seen after %d for segment %d while reading %s", position, prevPosition, segment, key);
-            readBuffer(buffer, reader, Version.fromVersion(userVersion));
-            prevSegment = segment;
-            prevPosition = position;
-        }
-    }
-
-    /**
-     * When using {@link PartitionRangeReadCommand} we need to work with {@link RowFilter} which works with columns.
-     * But the index doesn't care about table based queries and needs to be queried using the fields in the index, to
-     * support that this enum exists.  This enum represents the fields present in the index and can be used to apply
-     * filters to the index.
-     */
-    public enum SyntheticColumn
-    {
-        participants("participants", BytesType.instance),
-        store_id("store_id", Int32Type.instance),
-        txn_id("txn_id", BytesType.instance);
-
-        public final ColumnMetadata metadata;
-
-        SyntheticColumn(String name, AbstractType<?> type)
-        {
-            this.metadata = new ColumnMetadata("journal", "routes", new ColumnIdentifier(name, false), type, ColumnMetadata.NO_UNIQUE_ID, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR, null);
-        }
-    }
-
-    private class JournalTableRangeSearcher implements JournalRangeSearcher
-    {
-        private final Index tableIndex;
-
-        private JournalTableRangeSearcher()
-        {
-            this.tableIndex = cfs.indexManager.getIndexByName("record");
-            if (!cfs.indexManager.isIndexQueryable(tableIndex))
-                throw new AssertionError("Journal record index is not queryable");
-        }
-
-        @Override
-        public Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
-        {
-            CloseableIterator<TxnId> inMemory = index.search(commandStoreId, range, minTxnId, maxTxnId, decidedRX).results();
-            CloseableIterator<TxnId> table = tableSearch(commandStoreId, range.start(), range.end(), minTxnId, maxTxnId, decidedRX);
-            return new DefaultResult(minTxnId, maxTxnId, decidedRX, MergeIterator.get(Arrays.asList(inMemory, table)));
-        }
-
-        @Override
-        public Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
-        {
-            CloseableIterator<TxnId> inMemory = index.search(commandStoreId, key, minTxnId, maxTxnId, decidedRX).results();
-            CloseableIterator<TxnId> table = tableSearch(commandStoreId, key, minTxnId, maxTxnId);
-            return new DefaultResult(minTxnId, maxTxnId, decidedRX, MergeIterator.get(Arrays.asList(inMemory, table)));
-        }
-
-        private CloseableIterator<TxnId> tableSearch(int store, TokenKey start, TokenKey end, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
-        {
-            RowFilter rowFilter = RowFilter.create(false);
-            rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GT, OrderedRouteSerializer.serialize(start));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(end));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.txn_id.metadata, Operator.GTE, CommandSerializers.txnId.serialize(minTxnId));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.txn_id.metadata, Operator.LTE, CommandSerializers.timestamp.serialize(maxTxnId));
-            return process(store, rowFilter);
-        }
-
-        private CloseableIterator<TxnId> tableSearch(int store, TokenKey key, TxnId minTxnId, Timestamp maxTxnId)
-        {
-            RowFilter rowFilter = RowFilter.create(false);
-            rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GTE, OrderedRouteSerializer.serialize(key));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(key));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.txn_id.metadata, Operator.GTE, CommandSerializers.txnId.serialize(minTxnId));
-            rowFilter.add(AccordJournalTable.SyntheticColumn.txn_id.metadata, Operator.LTE, CommandSerializers.timestamp.serialize(maxTxnId));
-            return process(store, rowFilter);
-        }
-
-        private CloseableIterator<TxnId> process(int storeId, RowFilter rowFilter)
-        {
-            PartitionRangeReadCommand cmd = PartitionRangeReadCommand.create(cfs.metadata(),
-                                                                             FBUtilities.nowInSeconds(),
-                                                                             ColumnFilter.selectionBuilder()
-                                                                                         .add(AccordJournalTable.SyntheticColumn.store_id.metadata)
-                                                                                         .add(AccordJournalTable.SyntheticColumn.txn_id.metadata)
-                                                                                         .build(),
-                                                                             rowFilter,
-                                                                             DataLimits.NONE,
-                                                                             DataRange.allData(cfs.getPartitioner()));
-            Index.Searcher s = tableIndex.searcherFor(cmd);
-            try (ReadExecutionController controller = cmd.executionController())
-            {
-                UnfilteredPartitionIterator partitionIterator = s.search(controller);
-                return new CloseableIterator<>()
-                {
-
-                    @Override
-                    public void close()
-                    {
-                        partitionIterator.close();
-                    }
-
-                    @Override
-                    public boolean hasNext()
-                    {
-                        return partitionIterator.hasNext();
-                    }
-
-                    @Override
-                    public TxnId next()
-                    {
-                        UnfilteredRowIterator next = partitionIterator.next();
-                        JournalKey partitionKeyComponents = getJournalKey(next.partitionKey());
-                        Invariants.require(partitionKeyComponents.commandStoreId == storeId,
-                                              () -> String.format("table index returned a command store other than the exepcted one; expected %d != %d", storeId, partitionKeyComponents.commandStoreId));
-                        return partitionKeyComponents.id;
-                    }
-                };
-            }
-        }
-    }
-
-    /**
-     * Perform a read from Journal table, followed by the reads from all journal segments.
-     * <p>
-     * When reading from journal segments, skip descriptors that were read from the table.
-     */
-    public void readAll(K key, Reader reader)
-    {
-        readAll(key, new RecordConsumerAdapter<>(reader));
-    }
-
-    public void readAll(K key, RecordConsumer<K> reader)
-    {
-        try (OpOrder.Group readOrder = cfs.readOrdering.start())
-        {
-            // SELECT segments first, to avoid missing segments due to races compacting segment->sstable
-            Segments<K, V> segments = journal.segments();
-            try (TableKeyIterator table = readAllFromTable(key, readOrder))
-            {
-                boolean hasTableData = table.advance();
-                long minSegment = hasTableData ? table.segment : Long.MIN_VALUE;
-                // First, read all journal entries newer than anything flushed into sstables
-                Journal.readAll(key, (segment, position, key1, buffer, userVersion) -> {
-                    if (segment > minSegment)
-                        reader.accept(segment, position, key1, buffer, userVersion);
-                }, readOrder, segments);
-
-                // Then, read SSTables
-                while (hasTableData)
-                {
-                    reader.accept(table.segment, table.offset, key, table.value, table.userVersion);
-                    hasTableData = table.advance();
-                }
-            }
-        }
-    }
-
-    public void readLast(K key, Reader reader)
-    {
-        readLast(key, new RecordConsumerAdapter<>(reader));
-    }
-
-    public void readLast(K key, RecordConsumer<K> reader)
-    {
-        try (OpOrder.Group readOrder = cfs.readOrdering.start())
-        {
-            Segments<K, V> segments = journal.segments();
-            try (TableKeyIterator table = readAllFromTable(key, readOrder))
-            {
-                boolean hasTableData = table.advance();
-                long minSegment = hasTableData ? table.segment : Long.MIN_VALUE;
-
-                class JournalReader implements RecordConsumer<K>
-                {
-                    boolean read;
-                    @Override
-                    public void accept(long segment, int position, K key, ByteBuffer buffer, int userVersion)
-                    {
-                        if (segment > minSegment)
-                        {
-                            reader.accept(segment, position, key, buffer, userVersion);
-                            read = true;
-                        }
-                    }
-                }
-
-                // First, read all journal entries newer than anything flushed into sstables
-                JournalReader journalReader = new JournalReader();
-                Journal.readLast(key, journalReader, readOrder, segments);
-
-                // Then, read SSTables, if we haven't found a record already
-                if (hasTableData && !journalReader.read)
-                    reader.accept(table.segment, table.offset, key, table.value, table.userVersion);
-            }
-        }
-    }
-
-    // TODO (expected): why are recordColumn and versionColumn instance fields, so that this cannot be a static class?
-    class TableKeyIterator implements Closeable, RecordConsumer<K>
-    {
-        final K key;
-        final List<UnfilteredRowIterator> unmerged;
-        final UnfilteredRowIterator merged;
-
-        long segment;
-        int offset;
-        ByteBuffer value;
-        int userVersion;
-
-        TableKeyIterator(K key, List<UnfilteredRowIterator> unmerged, UnfilteredRowIterator merged)
-        {
-            this.key = key;
-            this.unmerged = unmerged;
-            this.merged = merged;
-        }
-
-        @Override
-        public void accept(long segment, int offset, K key, ByteBuffer buffer, int userVersion)
-        {
-            this.segment = segment;
-            this.offset = offset;
-            this.value = buffer;
-            this.userVersion = userVersion;
-        }
-
-        boolean advance()
-        {
-            if (merged == null || !merged.hasNext())
-                return false;
-
-            try
-            {
-                Row row = (Row) merged.next();
-                segment = LongType.instance.compose(ByteBuffer.wrap((byte[]) row.clustering().get(0)));
-                offset = Int32Type.instance.compose(ByteBuffer.wrap((byte[]) row.clustering().get(1)));
-                value = row.getCell(recordColumn).buffer();
-                userVersion = Int32Type.instance.compose(row.getCell(versionColumn).buffer());
-                return true;
-            }
-            catch (Throwable t)
-            {
-                throw new FSReadError("Failed to read from " + unmerged, t);
-            }
-        }
-
-        @Override
-        public void close()
-        {
-            if (merged != null)
-                merged.close();
-        }
-    }
-
-    private TableKeyIterator readAllFromTable(K key, OpOrder.Group readOrder)
-    {
-        DecoratedKey pk = JournalColumns.decorate(key);
-        List<UnfilteredRowIterator> iters = new ArrayList<>(3);
-        try
-        {
-            ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, pk));
-            for (SSTableReader sstable : view.sstables)
-            {
-                if (!sstable.mayContainAssumingKeyIsInRange(pk))
-                    continue;
-
-                UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs, sstable, pk, Slices.ALL, ColumnFilter.all(cfs.metadata()), false, NOOP_LISTENER);
-                if (iter.getClass() != EmptyIterators.EmptyUnfilteredRowIterator.class)
-                    iters.add(iter);
-            }
-
-            return new TableKeyIterator(key, iters, iters.isEmpty() ? null : UnfilteredRowIterators.merge(iters));
-        }
-        catch (Throwable t)
-        {
-            for (UnfilteredRowIterator iter : iters)
-            {
-                try { iter.close(); }
-                catch (Throwable t2) { t.addSuppressed(t2); }
-            }
-            throw t;
-        }
-    }
-
-    @SuppressWarnings("resource") // Auto-closeable iterator will release related resources
-    public CloseableIterator<Journal.KeyRefs<K>> keyIterator(@Nullable K min, @Nullable K max, boolean includeActive)
-    {
-        try (OpOrder.Group readOrder = cfs.readOrdering.start())
-        {
-            return new JournalAndTableKeyIterator(min, max, includeActive);
-        }
-    }
-
-    private class TableIterator extends AbstractIterator<K> implements CloseableIterator<K>
-    {
-        private final UnfilteredPartitionIterator mergeIterator;
-        private final RefViewFragment view;
-
-        private TableIterator(JournalKey min, JournalKey max)
-        {
-            Invariants.require((min != null && max != null) || min == max);
-            view = cfs.selectAndReference(View.select(SSTableSet.LIVE, r -> (max == null || JournalKey.SUPPORT.compare(getJournalKey(r.getFirst()), max) <= 0)
-                                                                         && (min == null || JournalKey.SUPPORT.compare(getJournalKey(r.getLast()), min) >= 0)));
-            List<ISSTableScanner> scanners = new ArrayList<>();
-            for (SSTableReader sstable : view.sstables)
-            {
-                if (min == null) scanners.add(sstable.getScanner());
-                else scanners.add(sstable.getScanner(new Bounds(JournalColumns.decorate(min), JournalColumns.decorate(max))));
-            }
-
-            mergeIterator = view.sstables.isEmpty()
-                            ? EmptyIterators.unfilteredPartition(cfs.metadata())
-                            : UnfilteredPartitionIterators.merge(scanners, UnfilteredPartitionIterators.MergeListener.NOOP);
-        }
-
-        @CheckForNull
-        protected K computeNext()
-        {
-            K ret = null;
-            if (mergeIterator.hasNext())
-            {
-                try (UnfilteredRowIterator partition = mergeIterator.next())
-                {
-                    ret = (K) getJournalKey(partition.partitionKey());
-                    while (partition.hasNext())
-                        partition.next();
-                }
-            }
-
-            if (ret != null)
-                return ret;
-            else
-                return endOfData();
-        }
-
-        @Override
-        public void close()
-        {
-            mergeIterator.close();
-            view.close();
-        }
-    }
-
-    private class JournalAndTableKeyIterator extends AbstractIterator<Journal.KeyRefs<K>> implements CloseableIterator<Journal.KeyRefs<K>>
-    {
-        final Journal<K, V>.SegmentKeyIterator journalIterator;
-        final TableIterator tableIterator;
-
-        private JournalAndTableKeyIterator(K min, K max, boolean includeActive)
-        {
-            // We must initialise journal reader first, else we may race with segment->table compaction and miss some data
-            // that is, the following sequence could happen:
-            //  - Select sstables to read
-            //  - Segments compacted; segments removed and sstables added
-            //  - Segment iterator created
-            // TODO (expected): segments should be sstables on creation
-            this.journalIterator = journal.segmentKeyIterator(min, max, includeActive ?  ignore -> true : Segment::isStatic);
-            this.tableIterator = new TableIterator(min, max);
-        }
-
-        K prevFromTable = null;
-        K prevFromJournal = null;
-
-        @Override
-        protected Journal.KeyRefs<K> computeNext()
-        {
-            K tableKey = tableIterator.hasNext() ? tableIterator.peek() : null;
-            K journalKey = journalIterator.hasNext() ? journalIterator.peek().key() : null;
-
-            if (journalKey != null)
-            {
-                Invariants.require(prevFromJournal == null || keySupport.compare(journalKey, prevFromJournal) >= 0, // == for case where we have not consumed previous on prev iteration
-                                   "Incorrect sort order in journal segments: %s should strictrly follow %s " + this, journalKey, prevFromJournal);
-                prevFromJournal = journalKey;
-            }
-            else
-            {
-                prevFromJournal = null;
-            }
-
-            if (tableKey != null)
-            {
-                Invariants.require(prevFromTable == null || keySupport.compare(tableKey, prevFromTable) >= 0, // == for case where we have not consumed previous on prev iteration
-                                   "Incorrect sort order in journal table: %s should strictrly follow %s " + this, tableKey, prevFromTable);
-                prevFromTable = tableKey;
-            }
-            else
-            {
-                prevFromTable = null;
-            }
-
-            if (tableKey == null)
-                return journalKey == null ? endOfData() : journalIterator.next();
-
-            if (journalKey == null)
-                return new Journal.KeyRefs<>(tableIterator.next());
-
-            int cmp = keySupport.compare(tableKey, journalKey);
-            if (cmp == 0)
-            {
-                tableIterator.next();
-                return journalIterator.next();
-            }
-
-            return cmp < 0 ? new Journal.KeyRefs<>(tableIterator.next()) : journalIterator.next();
-        }
-
-        public void close()
-        {
-            tableIterator.close();
-            journalIterator.close();
-        }
-    }
-
-    public static void readBuffer(ByteBuffer buffer, Reader reader, Version userVersion)
-    {
-        try (DataInputBuffer in = new DataInputBuffer(buffer, false))
-        {
-            reader.read(in, userVersion);
-        }
-        catch (IOException e)
-        {
-            // can only throw if serializer is buggy or bytes got corrupted
-            throw new RuntimeException(e);
-        }
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java

@@ -1,343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.io.IOException;
-import java.util.NavigableMap;
-
-import com.google.common.collect.ImmutableSortedMap;
-
-import accord.local.DurableBefore;
-import accord.local.RedundantBefore;
-import accord.primitives.Ranges;
-import accord.primitives.Timestamp;
-import accord.primitives.TxnId;
-
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.journal.AccordTopologyUpdate;
-import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers;
-import org.apache.cassandra.service.accord.serializers.Version;
-
-import static accord.local.CommandStores.RangesForEpoch;
-
-public class AccordJournalValueSerializers
-{
-    public interface FlyweightImage
-    {
-        void reset(JournalKey key);
-    }
-
-    public interface FlyweightSerializer<ENTRY, IMAGE extends FlyweightImage>
-    {
-        IMAGE mergerFor();
-
-        void serialize(JournalKey key, ENTRY from, DataOutputPlus out, Version userVersion) throws IOException;
-
-        void reserialize(JournalKey key, IMAGE from, DataOutputPlus out, Version userVersion) throws IOException;
-
-        void deserialize(JournalKey key, IMAGE into, DataInputPlus in, Version userVersion) throws IOException;
-
-        default IMAGE deserialize(JournalKey key, DataInputPlus in, Version userVersion) throws IOException
-        {
-            IMAGE image = mergerFor();
-            deserialize(key, image, in, userVersion);
-            return image;
-        }
-    }
-
-    public static class CommandDiffSerializer
-    implements FlyweightSerializer<AccordJournal.Writer, AccordJournal.Builder>
-    {
-        @Override
-        public AccordJournal.Builder mergerFor()
-        {
-            return new AccordJournal.Builder();
-        }
-
-        @Override
-        public void serialize(JournalKey key, AccordJournal.Writer writer, DataOutputPlus out, Version userVersion)
-        {
-            try
-            {
-                writer.write(out, userVersion);
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public void reserialize(JournalKey key, AccordJournal.Builder from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            from.serialize(out,
-                           // In CompactionIterator, we are dealing with relatively recent records, so we do not pass redundant before here.
-                           // However, we do on load and during Journal SSTable compaction.
-                           userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey journalKey, AccordJournal.Builder into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            into.deserializeNext(in, userVersion);
-        }
-    }
-
-    public abstract static class Accumulator<A, V> implements FlyweightImage
-    {
-        protected A accumulated;
-
-        public Accumulator(A initial)
-        {
-            this.accumulated = initial;
-        }
-
-        public void update(V newValue)
-        {
-            accumulated = accumulate(accumulated, newValue);
-        }
-
-        protected abstract A accumulate(A oldValue, V newValue);
-
-        public A get()
-        {
-            return accumulated;
-        }
-    }
-
-    public static class IdentityAccumulator<T> extends Accumulator<T, T>
-    {
-        final T initial;
-        boolean hasRead;
-        public IdentityAccumulator(T initial)
-        {
-            super(initial);
-            this.initial = initial;
-        }
-
-        @Override
-        public void reset(JournalKey key)
-        {
-            hasRead = false;
-            accumulated = initial;
-        }
-
-        @Override
-        protected T accumulate(T oldValue, T newValue)
-        {
-            if (hasRead)
-                return oldValue;
-            hasRead = true;
-            return newValue;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "IdentityAccumulator{" +
-                   initial +
-                   '}';
-        }
-    }
-
-    public static class RedundantBeforeSerializer
-    implements FlyweightSerializer<RedundantBefore, IdentityAccumulator<RedundantBefore>>
-    {
-        @Override
-        public IdentityAccumulator<RedundantBefore> mergerFor()
-        {
-            return new IdentityAccumulator<>(RedundantBefore.EMPTY);
-        }
-
-        @Override
-        public void serialize(JournalKey key, RedundantBefore entry, DataOutputPlus out, Version userVersion)
-        {
-            try
-            {
-                if (entry == RedundantBefore.EMPTY)
-                {
-                    out.writeInt(0);
-                    return;
-                }
-                out.writeInt(1);
-                CommandStoreSerializers.redundantBefore.serialize(entry, out);
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public void reserialize(JournalKey key, IdentityAccumulator<RedundantBefore> from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            serialize(key, from.get(), out, userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey journalKey, IdentityAccumulator<RedundantBefore> into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            if (in.readInt() == 0)
-            {
-                into.update(RedundantBefore.EMPTY);
-                return;
-            }
-            into.update(CommandStoreSerializers.redundantBefore.deserialize(in));
-        }
-    }
-
-    public static class DurableBeforeAccumulator extends Accumulator<DurableBefore, DurableBefore>
-    {
-        public DurableBeforeAccumulator()
-        {
-            super(DurableBefore.EMPTY);
-        }
-
-        @Override
-        public void reset(JournalKey key)
-        {
-            accumulated = DurableBefore.EMPTY;
-        }
-
-        @Override
-        protected DurableBefore accumulate(DurableBefore oldValue, DurableBefore newValue)
-        {
-            return DurableBefore.merge(oldValue, newValue);
-        }
-    }
-
-    public static class DurableBeforeSerializer
-    implements FlyweightSerializer<DurableBefore, DurableBeforeAccumulator>
-    {
-        public DurableBeforeAccumulator mergerFor()
-        {
-            return new DurableBeforeAccumulator();
-        }
-
-        @Override
-        public void serialize(JournalKey key, DurableBefore entry, DataOutputPlus out, Version userVersion)
-        {
-            try
-            {
-                CommandStoreSerializers.durableBefore.serialize(entry, out);
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public void reserialize(JournalKey key, DurableBeforeAccumulator from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            serialize(key, from.get(), out, userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey journalKey, DurableBeforeAccumulator into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            into.update(CommandStoreSerializers.durableBefore.deserialize(in));
-        }
-    }
-
-    public static class BootstrapBeganAtSerializer
-    implements FlyweightSerializer<NavigableMap<TxnId, Ranges>, IdentityAccumulator<NavigableMap<TxnId, Ranges>>>
-    {
-        @Override
-        public IdentityAccumulator<NavigableMap<TxnId, Ranges>> mergerFor()
-        {
-            return new IdentityAccumulator<>(ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY));
-        }
-
-        @Override
-        public void serialize(JournalKey key, NavigableMap<TxnId, Ranges> entry, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            CommandStoreSerializers.bootstrapBeganAt.serialize(entry, out);
-        }
-
-        @Override
-        public void reserialize(JournalKey key, IdentityAccumulator<NavigableMap<TxnId, Ranges>> image, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            serialize(key, image.get(), out, userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey key, IdentityAccumulator<NavigableMap<TxnId, Ranges>> into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            into.update(CommandStoreSerializers.bootstrapBeganAt.deserialize(in));
-        }
-    }
-
-    public static class SafeToReadSerializer
-    implements FlyweightSerializer<NavigableMap<Timestamp, Ranges>, IdentityAccumulator<NavigableMap<Timestamp, Ranges>>>
-    {
-        @Override
-        public IdentityAccumulator<NavigableMap<Timestamp, Ranges>> mergerFor()
-        {
-            return new IdentityAccumulator<>(ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY));
-        }
-
-        @Override
-        public void serialize(JournalKey key, NavigableMap<Timestamp, Ranges> from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            CommandStoreSerializers.safeToRead.serialize(from, out);
-        }
-
-        @Override
-        public void reserialize(JournalKey key, IdentityAccumulator<NavigableMap<Timestamp, Ranges>> from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            serialize(key, from.get(), out, userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey key, IdentityAccumulator<NavigableMap<Timestamp, Ranges>> into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            into.update(CommandStoreSerializers.safeToRead.deserialize(in));
-        }
-    }
-
-    public static class RangesForEpochSerializer
-    implements FlyweightSerializer<RangesForEpoch, Accumulator<RangesForEpoch, RangesForEpoch>>
-    {
-        public static final RangesForEpochSerializer instance = new RangesForEpochSerializer();
-        public IdentityAccumulator<RangesForEpoch> mergerFor()
-        {
-            return new IdentityAccumulator<>(null);
-        }
-
-        @Override
-        public void serialize(JournalKey key, RangesForEpoch from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            AccordTopologyUpdate.RangesForEpochSerializer.instance.serialize(from, out);
-        }
-
-        @Override
-        public void reserialize(JournalKey key, Accumulator<RangesForEpoch, RangesForEpoch> from, DataOutputPlus out, Version userVersion) throws IOException
-        {
-            serialize(key, from.get(), out, userVersion);
-        }
-
-        @Override
-        public void deserialize(JournalKey key, Accumulator<RangesForEpoch, RangesForEpoch> into, DataInputPlus in, Version userVersion) throws IOException
-        {
-            into.update(AccordTopologyUpdate.RangesForEpochSerializer.instance.deserialize(in));
-        }
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -240,7 +240,7 @@ public class AccordKeyspace
 
         public static ByteBuffer makeSystemTableKeyBytes(int commandStore, TokenKey key)
         {
-            ByteBuffer result = ByteBuffer.allocate(4 + TokenKey.serializer.serializedSizeWithoutPrefix(key));
+            ByteBuffer result = ByteBuffer.allocate(4 + TokenKey.serializer.serializedSizeWithoutPrefixOrLength(key));
             result.putInt(commandStore);
             TokenKey.serializer.serializeWithoutPrefixOrLength(key, result);
             result.flip();

src/java/org/apache/cassandra/service/accord/AccordMessageSink.java

@@ -47,6 +47,7 @@ import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.ResponseContext;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.service.TimeoutStrategy;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.utils.Clock;
 
 import static accord.messages.MessageType.StandardMessage.ACCEPT_REQ;

src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java

@@ -275,8 +275,14 @@ public class AccordObjectSizes
 
         size += dependencies.keyDeps.txnIdCount() * TIMESTAMP_SIZE;
         size += dependencies.rangeDeps.txnIdCount() * TIMESTAMP_SIZE;
-        size += KeyDeps.SerializerSupport.keysToTxnIds(dependencies.keyDeps).length * 4L;
-        size += RangeDeps.SerializerSupport.rangesToTxnIds(dependencies.rangeDeps).length * 4L;
+        if (dependencies.keyDeps.hasByKey())
+            size += KeyDeps.SerializerSupport.keysToTxnIds(dependencies.keyDeps).length * 4L;
+        if (dependencies.keyDeps.hasByTxnId())
+            size += KeyDeps.SerializerSupport.txnIdsToKeys(dependencies.keyDeps).length * 4L;
+        if (dependencies.rangeDeps.hasByRange())
+            size += RangeDeps.SerializerSupport.rangesToTxnIds(dependencies.rangeDeps).length * 4L;
+        if (dependencies.rangeDeps.hasByTxnId())
+            size += RangeDeps.SerializerSupport.txnIdsToRanges(dependencies.rangeDeps).length * 4L;
         return size;
     }
 

src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.NoSpamLogger.NoSpamLogStatement;

src/java/org/apache/cassandra/service/accord/AccordResult.java

@@ -21,6 +21,7 @@ package org.apache.cassandra.service.accord;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 
@@ -186,7 +187,7 @@ public class AccordResult<V> extends AsyncFuture<V> implements BiConsumer<V, Thr
             trySuccess((V) RetryWithNewProtocolResult.instance);
             return false;
         }
-        else if (fail instanceof RequestTimeoutException || fail instanceof TimeoutException)
+        else if (fail instanceof RequestTimeoutException || fail instanceof TimeoutException || fail instanceof CancellationException)
         {
             report = bookkeeping.newTimeout(txnId, keysOrRanges);
         }

src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java

@@ -42,9 +42,9 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
         }
 
         @Override
-        public void invalidate()
+        public void markUnsafe()
         {
-            super.invalidate();
+            super.markUnsafe();
             selfRef.release();
         }
 
@@ -54,7 +54,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
         }
     }
 
-    private boolean invalidated;
+    private boolean unsafe;
     private final AccordCacheEntry<TxnId, Command> global;
     private Command original;
     private Command current;
@@ -86,7 +86,7 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     public String toString()
     {
         return "AccordSafeCommand{" +
-               "invalidated=" + invalidated +
+               "invalidated=" + unsafe +
                ", global=" + global +
                ", original=" + original +
                ", current=" + current +
@@ -138,14 +138,14 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
     }
 
     @Override
-    public void invalidate()
+    public void markUnsafe()
     {
-        invalidated = true;
+        unsafe = true;
     }
 
     @Override
-    public boolean invalidated()
+    public boolean isUnsafe()
     {
-        return invalidated;
+        return unsafe;
     }
 }

src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java

@@ -37,6 +37,7 @@ import accord.local.Command;
 import accord.local.CommandStores;
 import accord.local.CommandSummaries;
 import accord.local.NodeCommandStoreService;
+import accord.local.RedundantBefore;
 import accord.local.cfk.CommandsForKey;
 import accord.local.cfk.SafeCommandsForKey;
 import accord.primitives.Timestamp;
@@ -48,6 +49,7 @@ import accord.utils.Invariants;
 import org.apache.cassandra.metrics.LogLinearDecayingHistograms;
 import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
 import org.apache.cassandra.service.accord.AccordCommandStore.SafeRedundantBefore;
+import org.apache.cassandra.service.accord.AccordDurableOnFlush.ReportDurable;
 import org.apache.cassandra.service.paxos.PaxosState;
 
 import static accord.utils.Invariants.illegalState;
@@ -189,6 +191,13 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         commandStore().rangeIndex().update(prev, updated, force);
     }
 
+    @Override
+    public void reportDurable(RedundantBefore addRedundantBefore, int flags)
+    {
+        upsertRedundantBefore(addRedundantBefore);
+        commandStore.maybeTerminated(ReportDurable.isCommandStoreFlush(flags), ReportDurable.isDataStoreFlush(flags));
+    }
+
     @Override
     public AccordCommandStore commandStore()
     {

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java

@@ -120,13 +120,13 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     }
 
     @Override
-    public void invalidate()
+    public void markUnsafe()
     {
         invalidated = true;
     }
 
     @Override
-    public boolean invalidated()
+    public boolean isUnsafe()
     {
         return invalidated;
     }

src/java/org/apache/cassandra/service/accord/AccordSafeState.java

@@ -23,8 +23,8 @@ public interface AccordSafeState<K, V> extends SafeState<V>
 {
     void set(V update);
     V original();
-    void invalidate();
-    boolean invalidated();
+    void markUnsafe();
+    boolean isUnsafe();
     void preExecute();
 
     AccordCacheEntry<K, V> global();
@@ -51,7 +51,7 @@ public interface AccordSafeState<K, V> extends SafeState<V>
 
     default void checkNotInvalidated()
     {
-        if (invalidated())
+        if (isUnsafe())
             throw new IllegalStateException("Cannot access invalidated " + this);
     }
 }

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -39,6 +40,7 @@ import javax.annotation.concurrent.GuardedBy;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.primitives.Ints;
 
+import org.agrona.collections.Long2LongHashMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,6 +55,8 @@ import accord.impl.SizeOfIntersectionSorter;
 import accord.impl.progresslog.DefaultProgressLog;
 import accord.impl.progresslog.DefaultProgressLogs;
 import accord.local.Catchup;
+import accord.local.CatchupHard;
+import accord.local.CommandStores;
 import accord.local.Node;
 import accord.local.Node.Id;
 import accord.local.ShardDistributor.EvenSplit;
@@ -76,18 +80,25 @@ import accord.topology.TopologyManager;
 import accord.topology.TopologyRange;
 import accord.utils.DefaultRandom;
 import accord.utils.Invariants;
+import accord.utils.Reduce;
+import accord.utils.UnhandledEnum;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 
 import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.config.AccordSpec;
+import org.apache.cassandra.config.AccordSpec.CatchupMode;
+import org.apache.cassandra.config.AccordSpec.JournalSpec.ReplaySavePoint;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.SystemKeyspace.BootstrapState;
+import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.journal.Descriptor;
 import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.AccordExecutorMetrics;
@@ -98,9 +109,10 @@ import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessageDelivery;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.repair.SharedContext;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
+import org.apache.cassandra.service.accord.AccordKeyspace.AccordColumnFamilyStores;
 import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
 import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutableKey;
@@ -111,8 +123,19 @@ import org.apache.cassandra.service.accord.api.AccordViolationHandler;
 import org.apache.cassandra.service.accord.api.CompositeTopologySorter;
 import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter;
 import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory;
+import org.apache.cassandra.service.accord.journal.AccordJournal;
+import org.apache.cassandra.service.accord.journal.ReplayMarkers;
 import org.apache.cassandra.service.accord.serializers.TableMetadatas;
 import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.topology.AccordFastPathCoordinator;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification;
+import org.apache.cassandra.service.accord.topology.AccordTopology;
+import org.apache.cassandra.service.accord.topology.AccordTopologyService;
+import org.apache.cassandra.service.accord.topology.EndpointMapping;
+import org.apache.cassandra.service.accord.topology.FetchTopologies;
+import org.apache.cassandra.service.accord.topology.WatermarkCollector;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnResult;
@@ -129,6 +152,7 @@ import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.concurrent.AsyncFuture;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Condition;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@@ -136,26 +160,33 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import static accord.api.Journal.TopologyUpdate;
 import static accord.api.ProtocolModifiers.Toggles.FastExec.MAY_BYPASS_SAFESTORE;
 import static accord.impl.progresslog.DefaultProgressLog.ModeFlag.CATCH_UP;
-import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE;
-import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE;
 import static accord.local.durability.DurabilityService.SyncLocal.Self;
 import static accord.local.durability.DurabilityService.SyncRemote.All;
 import static accord.messages.SimpleReply.Ok;
 import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
 import static accord.primitives.Txn.Kind.Write;
+import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorThreadTag.JOB;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.DAEMON;
+import static org.apache.cassandra.config.AccordSpec.CatchupMode.DISABLED;
+import static org.apache.cassandra.config.AccordSpec.CatchupMode.FALLBACK_TO_HARD;
+import static org.apache.cassandra.config.AccordSpec.CatchupMode.HARD;
+import static org.apache.cassandra.config.AccordSpec.JournalSpec.ReplayMode.RESET;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccord;
 import static org.apache.cassandra.config.DatabaseDescriptor.getAccordCommandStoreShardCount;
 import static org.apache.cassandra.config.DatabaseDescriptor.getAccordGlobalDurabilityCycle;
 import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityCycle;
 import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityMaxSplits;
 import static org.apache.cassandra.config.DatabaseDescriptor.getAccordShardDurabilityTargetSplits;
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.DRAIN;
 import static org.apache.cassandra.db.SystemKeyspace.BootstrapState.COMPLETED;
-import static org.apache.cassandra.journal.Params.ReplayMode.RESET;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadBookkeeping;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteBookkeeping;
-import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord;
+import static org.apache.cassandra.service.accord.topology.AccordTopology.tcmIdToAccord;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
@@ -253,7 +284,7 @@ public class AccordService implements IAccordService, Shutdownable
         ProtocolModifiers.Toggles.setDataStoreDetectsFutureReads(true);
     }
 
-    private enum State { INIT, STARTED, SHUTTING_DOWN, SHUTDOWN }
+    private enum State { INIT, STARTING, STARTED, STOPPED, SHUTTING_DOWN, SHUTDOWN }
 
     private final Node node;
     private final AccordMessageSink messageSink;
@@ -268,6 +299,7 @@ public class AccordService implements IAccordService, Shutdownable
 
     @GuardedBy("this")
     private volatile State state = State.INIT;
+    private final Condition isShutdown = Condition.newOneTimeCondition();
 
     private static final IAccordService NOOP_SERVICE = new NoOpAccordService();
 
@@ -275,17 +307,19 @@ public class AccordService implements IAccordService, Shutdownable
     //  tests can specify a DelegatingService if they want to override
     private static IAccordService instance;
     private static IAccordService unsafeInstance;
+    private static volatile IAccordService requestInstance;
+    private static volatile IAccordService replyInstance;
 
     @VisibleForTesting
     public static void unsafeSetNewAccordService(IAccordService service)
     {
-        unsafeInstance = instance = service;
+        unsafeInstance = instance = requestInstance = replyInstance = service;
     }
 
     @VisibleForTesting
     public static void unsafeSetNoop()
     {
-        unsafeInstance = instance = NOOP_SERVICE;
+        unsafeInstance = instance = requestInstance = replyInstance = NOOP_SERVICE;
     }
 
     public static IAccordService tryGetUnsafe()
@@ -317,14 +351,16 @@ public class AccordService implements IAccordService, Shutdownable
 
     public static IVerbHandler<? extends Request> requestHandlerOrNoop()
     {
-        if (instance == null) return ignore -> {};
-        return instance.requestHandler();
+        IAccordService accord = requestInstance;
+        if (accord == null) return ignore -> {};
+        return accord.requestHandler();
     }
 
     public static IVerbHandler<? extends Reply> responseHandlerOrNoop()
     {
-        if (unsafeInstance == null) return ignore -> {};
-        return unsafeInstance.responseHandler();
+        IAccordService accord = replyInstance;
+        if (accord == null) return ignore -> {};
+        return accord.responseHandler();
     }
 
     @VisibleForTesting
@@ -333,12 +369,12 @@ public class AccordService implements IAccordService, Shutdownable
         Invariants.require(instance == null);
         if (!DatabaseDescriptor.getAccordTransactionsEnabled())
         {
-            unsafeInstance = instance = NOOP_SERVICE;
+            unsafeSetNoop();
         }
         else
         {
             AccordService as = new AccordService(tcmIdToAccord(tcmId));
-            unsafeInstance = as;
+            unsafeInstance = replyInstance = as;
             as.localStartup();
         }
     }
@@ -349,11 +385,10 @@ public class AccordService implements IAccordService, Shutdownable
             return null;
 
         AccordService as = (AccordService) unsafeInstance;
-        if (as.state != State.INIT)
+        if (as.state != State.STARTING)
             return as;
 
         as.distributedStartupInternal();
-        instance = as;
 
         AccordReplicaMetrics.touch();
         AccordSystemMetrics.touch();
@@ -363,24 +398,25 @@ public class AccordService implements IAccordService, Shutdownable
     }
 
     @VisibleForTesting
-    public static boolean replayJournal(AccordService as)
+    private boolean replayJournal(Long2LongHashMap minSegments)
     {
+        if (getAccord().journal.replay == RESET)
+        {
+            if (getAccord().journal.replaySavePoint != ReplaySavePoint.NO)
+                throw new IllegalArgumentException("Cannot reset state and replay from a save point; must modify either accord.journal.replay or accord.journal.replay_save_point");
+            AccordKeyspace.truncateCommandsForKey();
+        }
+
         logger.info("Starting journal replay.");
         long start = nanoTime();
-        if (as.journalConfiguration().replayMode() == RESET)
-            AccordKeyspace.truncateCommandsForKey();
-
-        as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop());
-        as.journal().replay(as.node.commandStores());
+        boolean success = journal().replay(node.commandStores(), minSegments);
         logger.info("Waiting for command stores to quiesce.");
-        ((AccordCommandStores)as.node.commandStores()).waitForQuiescence();
-        getBlocking(as.node.commandStores().forAll("Post Replay", safeStore -> ((AccordCommandStore)safeStore.commandStore()).rangeIndex().postReplay()));
-        as.journal.unsafeSetStarted();
-        as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start());
+        ((AccordCommandStores)node.commandStores()).waitForQuiescence();
+        getBlocking(node.commandStores().forAll("Post Replay", safeStore -> ((AccordCommandStore)safeStore.commandStore()).rangeIndex().postReplay()));
 
         long end = nanoTime();
         logger.info("Finished journal replay. {}s elapsed", String.format("%.2f", NANOSECONDS.toMillis(end - start)/1000.0));
-        return true;
+        return success;
     }
 
     public static IAccordService instance()
@@ -410,7 +446,7 @@ public class AccordService implements IAccordService, Shutdownable
         Invariants.require(localId != null, "static localId must be set before instantiating AccordService");
         logger.info("Starting accord with nodeId {}", localId);
         AccordAgent agent = FBUtilities.construct(CassandraRelevantProperties.ACCORD_AGENT_CLASS.getString(AccordAgent.class.getName()), "AccordAgent");
-        agent.setNodeId(localId);
+        agent.setup(localId);
         AccordTimeService time = new AccordTimeService();
         final RequestCallbacks callbacks = new RequestCallbacks(time);
         this.scheduler = new AccordScheduler();
@@ -449,39 +485,127 @@ public class AccordService implements IAccordService, Shutdownable
         if (state != State.INIT)
             return;
 
+        boolean rebootstrap = false;
+        {
+            long startMarker = ReplayMarkers.readStartMarker();
+            long stopMarker = ReplayMarkers.readStopMarker();
+            if (stopMarker < startMarker)
+            {
+                switch (getAccord().journal.stopMarkerFailurePolicy)
+                {
+                    default: throw new UnhandledEnum(getAccord().journal.stopMarkerFailurePolicy);
+                    case EXIT:
+                        throw new RuntimeException("Stop marker is older than start marker (" + stopMarker + '<' + startMarker + ") , so cannot assume we have a complete log of our votes in any consensus groups. Exiting.");
+
+                    case UNSAFE_STARTUP:
+                        logger.warn("Stop marker is older than start marker ({}<{}), so cannot assume we have a complete log of our votes in any consensus groups. Continuing to startup as configured.", stopMarker, startMarker);
+                        break;
+
+                    case REBOOTSTRAP:
+                        logger.info("Stop marker is older than start marker ({}<{}). Rebootstrapping.", stopMarker, startMarker);
+                        rebootstrap = true;
+                }
+            }
+        }
+
+        logger.info("Starting background compaction of system_accord");
+        // We control this ourselves to ensure it starts when we need it, as especially commands_for_key
+        // can accumulate a lot of state and degrade replay performance significantly
+        scheduler.recurring(() -> {
+            CompactionManager.instance.submitBackground(AccordColumnFamilyStores.commandsForKey);
+            CompactionManager.instance.submitBackground(AccordColumnFamilyStores.journal);
+        }, 1L, MINUTES);
+
+        state = State.STARTING;
         node.unsafeSetReplaying(true);
         try
         {
-            journal.start(node);
-            node.load();
+            node.durability().stop();
 
+            journal.open(node);
+            node.load();
 
             ClusterMetadata metadata = ClusterMetadata.current();
             endpointMapper.updateMapping(metadata);
 
-            List<TopologyUpdate> images = journal.replayTopologies();
-            if (!images.isEmpty())
-            {
-                // Initialise command stores using latest topology from the log;
-                // if there are no local command stores, don't report any topologies and simply fetch the latest known in the cluster
-                // this avoids a registered (not joined) node learning of topologies, then later restarting with some intervening
-                // epochs having been garbage collected by the other nodes in the cluster
-                TopologyUpdate last = images.get(images.size() - 1);
-                if (!last.commandStores.isEmpty())
-                {
-                    node.commandStores().initializeTopologyUnsafe(last);
+            // Initialise command stores using latest topology from the log;
+            // if there are no local command stores, don't report any topologies and simply fetch the latest known in the cluster
+            // this avoids a registered (not joined) node learning of topologies, then later restarting with some intervening
+            // epochs having been garbage collected by the other nodes in the cluster
 
-                    // Replay local epochs
-                    for (TopologyUpdate image : images)
-                        node.topology().reportTopology(image.global);
+            topologyService.onStartup(node);
+            List<TopologyUpdate> images = journal.loadTopologies();
+            TopologyUpdate last = images.isEmpty() ? null : images.get(images.size() - 1);
+            boolean initialiseCommandStores = last != null && !last.commandStores.isEmpty();
+            if (initialiseCommandStores)
+                node.commandStores().initializeTopologyUnsafe(last);
+
+            node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop());
+
+            // restore save points before starting the journal so we can validate consistency between journal and save point state (where possible)
+            Long2LongHashMap minSegments;
+            if (rebootstrap) minSegments = null;
+            else
+            {
+                switch (getAccord().journal.replaySavePoint)
+                {
+                    default: throw new UnhandledEnum(getAccord().journal.replaySavePoint);
+                    case NO: minSegments = new Long2LongHashMap(0L); break;
+                    case LATEST: minSegments = restoreFromSavePoints(node.commandStores());
                 }
             }
-            replayJournal(this);
+
+            // now start the journal before we replay, as replay may trigger its own new journal writes
+            journal.start(node);
+
+            if (initialiseCommandStores)
+            {
+                // Replay local topologies
+                for (TopologyUpdate image : images)
+                    node.topology().reportTopology(image.global);
+            }
+
+            node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start());
+            if (rebootstrap)
+            {
+                // rebootstrap expects the durability service to process visibility sync points for command stores
+                node.durability().start();
+                getBlocking(node.commandStores().rebootstrap(node));
+            }
+            else
+            {
+                replayJournal(minSegments);
+
+                logger.info("Try to execute pending transactions...");
+                List<AsyncResult<Void>> results = new ArrayList<>();
+                node.commandStores().forAllUnsafe(commandStore -> results.add(commandStore.tryToExecuteListeningTxns(false)));
+                if (!results.isEmpty())
+                    getBlocking(AsyncResults.reduce(results, Reduce.toNull()));
+            }
         }
         finally
         {
             node.unsafeSetReplaying(false);
         }
+        node.commandStores().forAllUnsafe(commandStore -> ((AccordCommandStore)commandStore).ensureDurable());
+    }
+
+    private Long2LongHashMap restoreFromSavePoints(CommandStores commandStores)
+    {
+        Long2LongHashMap result = new Long2LongHashMap(0);
+        Future<List<Map.Entry<Integer, Long>>> future = toFuture(((AccordCommandStores)commandStores).restoreState());
+        future.awaitThrowUncheckedOnInterrupt();
+        List<Map.Entry<Integer, Long>> success = future.getNow();
+        if (success != null)
+        {
+            for (Map.Entry<Integer, Long> e : success)
+            {
+                if (e != null && e.getValue() > 0)
+                    result.put(e.getKey(), e.getValue());
+            }
+        }
+
+        return result;
     }
 
     private void distributedStartupInternal()
@@ -494,29 +618,34 @@ public class AccordService implements IAccordService, Shutdownable
 
         // we set ourselves to STARTED before starting progress logs as this is the condition we use to decide if we
         // start the progress log on command store initialisation (so creates a synchronisation point)
+        journal.writeStartMarker();
         state = State.STARTED;
-        node.commandStores().forAll("", safeStore -> safeStore.progressLog().start());
-
-        // trigger catchup only after our progress mechanisms are initialised
-        catchup();
+        instance = requestInstance = this;
+        node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start());
 
         node.durability().shards().reconfigure(Ints.checkedCast(getAccordShardDurabilityTargetSplits()),
                                                Ints.checkedCast(getAccordShardDurabilityMaxSplits()),
                                                Ints.checkedCast(getAccordShardDurabilityCycle(SECONDS)), SECONDS);
         node.durability().global().setGlobalCycleTime(Ints.checkedCast(getAccordGlobalDurabilityCycle(SECONDS)), SECONDS);
-        // Only enable durability scheduling and progress logs _after_ we have fully replayed journal
-        node.durability().start();
+
+        // Only enable durability scheduling
+        if (!node.durability().isStarted())
+            node.durability().start();
+
+        // trigger catchup only after our progress mechanisms are initialised
+        catchup();
     }
 
     void catchup()
     {
         AccordSpec spec = DatabaseDescriptor.getAccord();
-        if (!spec.catchup_on_start)
+        if (spec.catchup_on_start == DISABLED)
         {
             logger.info("Catchup disabled; continuing to startup");
             return;
         }
 
+        CatchupMode mode = spec.catchup_on_start;
         BootstrapState bootstrapState = SystemKeyspace.getBootstrapState();
         if (bootstrapState == COMPLETED)
         {
@@ -527,10 +656,21 @@ public class AccordService implements IAccordService, Shutdownable
                 int attempts = 1;
                 while (true)
                 {
-                    logger.info("Catchup with quorum...");
+                    logger.info("Catchup ({}) with quorum...", mode);
                     long start = nanoTime();
                     long failAt = start + maxLatencyNanos;
-                    Future<Void> f = toFuture(Catchup.catchup(node));
+                    Future<Void> f;
+                    {
+                        AsyncChain<Void> submit;
+                        if (mode == HARD)
+                        {
+                            if (!node.durability().isStarted())
+                                node.durability().start();
+                            submit = CatchupHard.catchup(node);
+                        }
+                        else submit = Catchup.catchup(node);
+                        f = toFuture(submit);
+                    }
                     if (!f.awaitUntilThrowUncheckedOnInterrupt(failAt))
                     {
                         if (spec.catchup_on_start_exit_on_failure)
@@ -572,6 +712,8 @@ public class AccordService implements IAccordService, Shutdownable
                     }
 
                     logger.info("Catchup was slow, so we may behind again; retrying");
+                    if (mode == FALLBACK_TO_HARD)
+                        mode = HARD;
                 }
             }
             finally
@@ -978,14 +1120,15 @@ public class AccordService implements IAccordService, Shutdownable
         return scheduler.isTerminated();
     }
 
+    static class FlushingCacheEntries extends AsyncResults.CountingResult implements Runnable
+    {
+        public FlushingCacheEntries() { super(1); }
+        @Override public void run() { decrement(); }
+    }
+
     public synchronized Future<Void> flushCaches()
     {
-        class Ready extends AsyncResults.CountingResult implements Runnable
-        {
-            public Ready() { super(1); }
-            @Override public void run() { decrement(); }
-        }
-        Ready ready = new Ready();
+        FlushingCacheEntries flushing = new FlushingCacheEntries();
         AccordCommandStores commandStores = (AccordCommandStores) node.commandStores();
         commandStores.forAllUnsafe(unsafeStore -> {
             AccordCommandStore commandStore = (AccordCommandStore)unsafeStore;
@@ -994,34 +1137,87 @@ public class AccordService implements IAccordService, Shutdownable
                 caches.commandsForKeys().forEach(entry -> {
                     if (entry.isModified())
                     {
-                        ready.increment();
-                        caches.global().saveWhenReadyExclusive(entry, ready);
+                        flushing.increment();
+                        caches.global().saveWhenReadyExclusive(entry, flushing);
                     }
                 });
             }
         });
-        ready.decrement();
-        AsyncPromise<Void> result = new AsyncPromise<>();
-        ready.invoke((success, fail) -> {
-            if (fail != null) result.tryFailure(fail);
-            else result.trySuccess(null);
-        });
-        return result;
+        flushing.decrement();
+        return toFuture(flushing);
     }
 
-    public synchronized void markShuttingDown()
+    public synchronized void stop()
     {
-        state = State.SHUTTING_DOWN;
+        if (state == State.INIT)
+            return;
+
+        logger.info("Stopping Accord");
+        requestInstance = replyInstance = null;
+        node.durability().stop();
+        // TODO (expected): stop TopologyManager from reporting new topologies
+        topologyService.shutdown();
+        AccordCommandStores commandStores = (AccordCommandStores)node.commandStores();
+        Set<TableId> tableIds = commandStores.shutdownStores();
+        commandStores.waitForQuiescence();
+        journal.writeSafeStopMarker();
+        scheduler.shutdownNow();
+        toFuture(flushCaches()).map(ignore -> {
+            return AccordColumnFamilyStores.commandsForKey.forceFlush(DRAIN);
+        });
+        for (TableId tableId : tableIds)
+        {
+            ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
+            if (cfs != null)
+                cfs.forceFlush(DRAIN);
+        }
+
+        state = State.STOPPED;
+        logger.info("Stopped Accord");
     }
 
     @Override
     public synchronized void shutdown()
     {
-        if (state != State.STARTED && state != State.SHUTTING_DOWN)
+        if (state.compareTo(State.SHUTDOWN) >= 0)
             return;
+
+        if (state.compareTo(State.STOPPED) < 0)
+            stop();
+
+        logger.info("Shutting down Accord");
         state = State.SHUTTING_DOWN;
-        shutdownAndWait(1, TimeUnit.MINUTES);
-        state = State.SHUTDOWN;
+        long deadlineNanos = nanoTime() + DatabaseDescriptor.getAccord().shutdown_grace_period.toDuration().toNanos();
+        executorFactory().startThread("ShutdownAccord", () -> {
+            AccordCommandStores commandStores = (AccordCommandStores)node.commandStores();
+            boolean safeShutdown = commandStores.awaitStoreTermination(deadlineNanos);
+            if (!safeShutdown)
+                logger.warn("Cannot write safe replay marker as not all command stores terminated promptly");
+
+            commandStores.waitForQuiescence();
+            Descriptor lastSegment = journal.stop();
+            if (lastSegment == null && safeShutdown)
+                logger.warn("Cannot write safe replay marker as no segment descriptor reported by journal");
+
+            if (safeShutdown && lastSegment != null)
+            {
+                Future<Boolean> save = toFuture(commandStores.saveState(lastSegment));
+                if (!save.awaitUntilThrowUncheckedOnInterrupt(deadlineNanos))
+                    logger.error("Timeout waiting to write safe replay markers");
+                else if (save.cause() != null)
+                    logger.error("Failed to write some safe replay markers", save.cause());
+                else if (!save.getNow())
+                    logger.error("Failed to write some safe replay markers");
+                else
+                    logger.info("Written safe replay markers");
+            }
+
+            commandStores.shutdownExecutors();
+            journal.close();
+            state = State.SHUTDOWN;
+            isShutdown.signalAll();
+            logger.info("Accord Shutdown");
+        }, DAEMON, JOB);
     }
 
     @Override
@@ -1034,32 +1230,32 @@ public class AccordService implements IAccordService, Shutdownable
     @Override
     public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
     {
-        try
+        long deadlineNanos = nanoTime() + units.toNanos(timeout);
+        boolean success = true;
+        if (!isShutdown.awaitUntil(deadlineNanos))
         {
-            ExecutorUtils.awaitTermination(timeout, units, shutdownableSubsystems());
-            return true;
+            logger.error("Accord command stores did not terminate before timeout elapsed");
+            success = false;
         }
-        catch (TimeoutException e)
+        try { ExecutorUtils.awaitTerminationUntil(deadlineNanos, Arrays.asList(scheduler, node.commandStores())); }
+        catch (InterruptedException | TimeoutException e)
         {
-            return false;
+            logger.error("Accord executors did not terminate before timeout elapsed");
+            success = false;
         }
-    }
-
-    private List<Shutdownable> shutdownableSubsystems()
-    {
-        return Arrays.asList((AccordCommandStores)node.commandStores(), journal, topologyService, scheduler);
+        if (!journal.awaitTerminationUntil(deadlineNanos))
+        {
+            logger.error("Accord journal did not terminate before timeout elapsed");
+            success = false;
+        }
+        return success;
     }
 
     @VisibleForTesting
     @Override
-    public void shutdownAndWait(long timeout, TimeUnit unit)
+    public void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
     {
-        if (!ExecutorUtils.shutdownThenWait(shutdownableSubsystems(), timeout, unit))
-            logger.error("One or more subsystems did not shut down cleanly.");
-
-        node.commandStores().forAllUnsafe(commandStore -> {
-            logger.info("{} stopping with durability: {}", commandStore, commandStore.unsafeGetRedundantBefore().map(b -> b == null ? null : b.maxBoundBoth(LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE), TxnId[]::new));
-        });
+        ExecutorUtils.shutdownAndWait(timeout, unit, this);
     }
 
     @Override
@@ -1120,11 +1316,7 @@ public class AccordService implements IAccordService, Shutdownable
     public static void receive(MessageDelivery sink, TopologyManager topologyManager, Message<Notification> message)
     {
         AccordSyncPropagator.Notification notification = message.payload;
-        notification.readyToCoordinate.forEach(id -> topologyManager.onReadyToCoordinate(id, notification.epoch));
-        if (!notification.closed.isEmpty())
-            topologyManager.onEpochClosed(notification.closed, notification.epoch);
-        if (!notification.retired.isEmpty())
-            topologyManager.onEpochRetired(notification.retired, notification.epoch);
+        notification.process(topologyManager);
         sink.respond(Ok, message);
     }
 

src/java/org/apache/cassandra/service/accord/AccordTask.java

@@ -90,7 +90,6 @@ import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_ACCO
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
 import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED;
 import static org.apache.cassandra.service.accord.AccordTask.State.FAILED;
-import static org.apache.cassandra.service.accord.AccordTask.State.FAILING;
 import static org.apache.cassandra.service.accord.AccordTask.State.FINISHED;
 import static org.apache.cassandra.service.accord.AccordTask.State.INITIALIZED;
 import static org.apache.cassandra.service.accord.AccordTask.State.LOADING;
@@ -164,10 +163,9 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
         WAITING_TO_RUN(INITIALIZED, SCANNING_RANGES, WAITING_TO_LOAD, LOADING),
         RUNNING(WAITING_TO_RUN),
         PERSISTING(RUNNING),
-        FAILING(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, RUNNING, PERSISTING),
         FINISHED(RUNNING, PERSISTING),
         CANCELLED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN),
-        FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, RUNNING, PERSISTING, FAILING);
+        FAILED(WAITING_TO_SCAN_RANGES, SCANNING_RANGES, WAITING_TO_LOAD, LOADING, WAITING_TO_RUN, RUNNING, PERSISTING);
 
         private final int permittedFrom;
 
@@ -739,46 +737,28 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
 
     public void fail(Throwable throwable)
     {
-        commandStore.agent().onException(throwable);
         if (state.isComplete())
             return;
 
-        if (commandStore.hasSafeStore())
-            commandStore.agent().onException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
-
-        state(FAILING);
-        if (callback != null)
-            callback.accept(null, throwable);
-    }
-
-    public void failExclusive(Throwable throwable)
-    {
-        boolean newFailure = state != FAILING;
         try
         {
-            if (newFailure)
-            {
-                commandStore.agent().onException(throwable);
-                if (state.isComplete())
-                    return;
-
-                if (commandStore.hasSafeStore())
-                    commandStore.agent().onException(new IllegalStateException(String.format("Failure to cleanup safe store for %s; status=%s", this, state), throwable));
-            }
-
+            commandStore.agent().onException(throwable);
             state(FAILED);
         }
         finally
         {
-            if (newFailure && callback != null)
+            if (callback != null)
                 callback.accept(null, throwable);
         }
     }
 
+    public void failExclusive(Throwable throwable)
+    {
+        fail(throwable);
+    }
+
     protected void cleanupExclusive()
     {
-        if (state == FAILING)
-            state(FAILED);
         Invariants.expect(state.isExecuted());
         releaseResources(commandStore.cachesExclusive());
         if (runningAt != 0)
@@ -815,7 +795,7 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
         if (rangeScanner != null)
             rangeScanner.cancelled = true;
         if (callback != null)
-            callback.accept(null, new CancellationException());
+            commandStore.executor().submit(() -> callback.accept(null, new CancellationException()));
     }
 
     void cancelExclusive(AccordExecutor owner)
@@ -912,7 +892,7 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
     {
         for (AccordSafeState<K, V> safeState : map.values())
         {
-            if (safeState.invalidated()) continue;
+            if (safeState.isUnsafe()) continue;
             try { cache.release(safeState, this); }
             catch (Throwable t) { suppressedBy.addSuppressed(t); }
         }
@@ -922,7 +902,7 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
     {
         for (AccordSafeState<?, ?> safeState : map.values())
         {
-            if (safeState.invalidated()) continue;
+            if (safeState.isUnsafe()) continue;
             try { cache.release(safeState, this); }
             catch (Throwable t) { suppressedBy.addSuppressed(t); }
         }
@@ -1144,7 +1124,8 @@ public abstract class AccordTask<R> extends SubmittableTask implements Function<
 
         void cleanup(Caches caches)
         {
-            loader.cleanupExclusive(caches);
+            if (loader != null)
+                loader.cleanupExclusive(caches);
         }
 
         CommandSummaries finish(Caches caches)

src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java

@@ -28,6 +28,7 @@ import accord.messages.Request;
 
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.utils.NoSpamLogger;
 
 public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -61,9 +61,12 @@ import org.apache.cassandra.journal.Params;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
 import org.apache.cassandra.service.accord.api.AccordScheduler;
 import org.apache.cassandra.service.accord.api.AccordTopologySorter;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator;
+import org.apache.cassandra.service.accord.topology.AccordSyncPropagator.Notification;
+import org.apache.cassandra.service.accord.topology.AccordTopologyService;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
@@ -122,7 +125,7 @@ public interface IAccordService
     void localStartup();
 
     Future<Void> flushCaches();
-    void markShuttingDown();
+    void stop();
     void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException;
 
     AccordScheduler scheduler();
@@ -289,7 +292,7 @@ public interface IAccordService
         }
 
         @Override
-        public void markShuttingDown()
+        public void stop()
         {
         }
 
@@ -498,9 +501,9 @@ public interface IAccordService
         }
 
         @Override
-        public void markShuttingDown()
+        public void stop()
         {
-            delegate.markShuttingDown();
+            delegate.stop();
         }
 
         @Override

src/java/org/apache/cassandra/service/accord/InMemoryRangeIndex.java

@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.List;
@@ -27,6 +28,7 @@ import java.util.function.BooleanSupplier;
 
 import javax.annotation.Nullable;
 
+import accord.impl.cfr.IdEntry;
 import accord.impl.cfr.InMemoryRangeSummaryIndex;
 import accord.impl.cfr.LoadListener;
 import accord.local.Command;
@@ -41,6 +43,12 @@ import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.utils.async.Cancellable;
 
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers;
+
+import static org.apache.cassandra.io.util.CompressedFrameDataInputPlus.readList;
+import static org.apache.cassandra.io.util.CompressedFrameDataOutputPlus.writeList;
+
 public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements RangeIndex
 {
     public static class Loader extends RangeIndex.Loader
@@ -122,7 +130,7 @@ public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements Ran
         }
 
         @Override
-        AccordCommandStore commandStore()
+        protected AccordCommandStore commandStore()
         {
             return owner.commandStore;
         }
@@ -152,4 +160,22 @@ public class InMemoryRangeIndex extends InMemoryRangeSummaryIndex implements Ran
     {
         return new Loader(this, redundantBefore, maxDecidedRX, primaryTxnId, searchKeysOrRanges, testKind, minTxnId, maxTxnId, loadKeysFor);
     }
+
+    @Override
+    public void save(File file) throws IOException
+    {
+        writeList(file, snapshot(), CommandStoreSerializers.rangeIndexIdEntry);
+    }
+
+    @Override
+    public List<IdEntry> load(File file) throws IOException
+    {
+        return readList(file, CommandStoreSerializers.rangeIndexIdEntry);
+    }
+
+    @Override
+    public void restore(Object loaded)
+    {
+        restore((List<IdEntry>)loaded);
+    }
 }

src/java/org/apache/cassandra/service/accord/JournalKey.java

@@ -31,22 +31,25 @@ import accord.utils.Invariants;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.journal.KeySupport;
-import org.apache.cassandra.service.accord.journal.AccordTopologyUpdate;
+import org.apache.cassandra.service.accord.journal.MergeSerializer;
 import org.apache.cassandra.utils.ByteArrayUtil;
 
 import static org.apache.cassandra.db.TypeSizes.BYTE_SIZE;
 import static org.apache.cassandra.db.TypeSizes.INT_SIZE;
 import static org.apache.cassandra.db.TypeSizes.LONG_SIZE;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.BootstrapBeganAtSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.CommandDiffSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.DurableBeforeSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.RangesForEpochSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.RedundantBeforeSerializer;
-import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.SafeToReadSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.BootstrapBeganAtSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.CommandChangeSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.DurableBeforeSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.RangesForEpochSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.RedundantBeforeSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.SafeToReadSerializer;
+import static org.apache.cassandra.service.accord.journal.MergeSerializers.TopologySerializer;
 
 public final class JournalKey
 {
+    // TODO (expected): do we need a dedicated buffer here?
+    private static final ThreadLocal<byte[]> keyCRCBytes = ThreadLocal.withInitial(() -> new byte[JournalKeySupport.TOTAL_SIZE]);
+
     public final Type type;
     public final TxnId id;
     public final int commandStoreId;
@@ -186,7 +189,7 @@ public final class JournalKey
         @Override
         public void updateChecksum(Checksum crc, JournalKey key, int userVersion)
         {
-            byte[] out = AccordJournal.keyCRCBytes.get();
+            byte[] out = keyCRCBytes.get();
             serialize(key, out);
             crc.update(out, 0, out.length);
         }
@@ -229,7 +232,7 @@ public final class JournalKey
             if (cmp == 0) cmp = k1.id.compareTo(k2.id);
             return cmp;
         }
-    };
+    }
 
     @Override
     public boolean equals(Object other)
@@ -264,20 +267,20 @@ public final class JournalKey
 
     public enum Type
     {
-        COMMAND_DIFF                 (0, new CommandDiffSerializer(), true),
+        COMMAND_DIFF                 (0, new CommandChangeSerializer(), true),
         REDUNDANT_BEFORE             (1, new RedundantBeforeSerializer(), false),
         DURABLE_BEFORE               (2, new DurableBeforeSerializer(), false),
         SAFE_TO_READ                 (3, new SafeToReadSerializer(), false),
         BOOTSTRAP_BEGAN_AT           (4, new BootstrapBeganAtSerializer(), false),
         RANGES_FOR_EPOCH             (5, new RangesForEpochSerializer(), false),
-        TOPOLOGY_UPDATE              (6, new AccordTopologyUpdate.FlyweightSerializer(), true),
+        TOPOLOGY_UPDATE              (6, new TopologySerializer(), true),
         ;
 
         public final int id;
-        public final FlyweightSerializer<?, ?> serializer;
+        public final MergeSerializer<?, ?, ?> serializer;
         public final boolean usesTxnId;
 
-        Type(int id, FlyweightSerializer<?, ?> serializer, boolean usesTxnId)
+        Type(int id, MergeSerializer<?, ?, ?> serializer, boolean usesTxnId)
         {
             this.id = id;
             this.serializer = serializer;

src/java/org/apache/cassandra/service/accord/RangeIndex.java

@@ -28,6 +28,7 @@ import accord.local.CommandSummaries;
 import accord.local.LoadKeysFor;
 import accord.local.MaxDecidedRX;
 import accord.local.RedundantBefore;
+import accord.primitives.Ranges;
 import accord.primitives.Routable;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
@@ -37,6 +38,8 @@ import accord.utils.Invariants;
 
 import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.accord.journal.CommandChanges;
 import org.apache.cassandra.service.accord.serializers.Version;
 
 import static accord.api.Journal.Load.MINIMAL;
@@ -52,12 +55,12 @@ public interface RangeIndex
             super(redundantBefore, maxDecidedRX, primaryTxnId, searchKeysOrRanges, testKinds, minTxnId, maxTxnId, loadKeysFor);
         }
 
-        abstract AccordCommandStore commandStore();
+        protected abstract AccordCommandStore commandStore();
 
-        abstract void loadExclusive(Map<Timestamp, CommandSummaries.Summary> into, AccordCommandStore.Caches caches);
-        abstract void load(Map<Timestamp, CommandSummaries.Summary> into, BooleanSupplier abort);
-        abstract void finish(Map<Timestamp, CommandSummaries.Summary> into);
-        abstract void cleanupExclusive(AccordCommandStore.Caches caches);
+        protected abstract void loadExclusive(Map<Timestamp, CommandSummaries.Summary> into, AccordCommandStore.Caches caches);
+        protected abstract void load(Map<Timestamp, CommandSummaries.Summary> into, BooleanSupplier abort);
+        protected abstract void finish(Map<Timestamp, CommandSummaries.Summary> into);
+        protected abstract void cleanupExclusive(AccordCommandStore.Caches caches);
 
         protected CommandSummaries.Summary loadFromDisk(TxnId txnId)
         {
@@ -110,7 +113,7 @@ public interface RangeIndex
                 return ifRelevant((Command) command);
 
             Invariants.require(command instanceof ByteBuffer);
-            AccordJournal.Builder builder = new AccordJournal.Builder(txnId, loadKeysFor != RECOVERY ? MINIMAL : MINIMAL_WITH_DEPS);
+            CommandChanges builder = new CommandChanges(txnId, loadKeysFor != RECOVERY ? MINIMAL : MINIMAL_WITH_DEPS);
             ByteBuffer buffer = (ByteBuffer) command;
             buffer.mark();
             try (DataInputBuffer buf = new DataInputBuffer(buffer, false))
@@ -135,6 +138,10 @@ public interface RangeIndex
     }
 
     Loader loader(TxnId primaryTxnId, Timestamp primaryExecuteAt, LoadKeysFor loadKeysFor, Unseekables<?> keysOrRanges);
-    void update(Command prev, Command updated, boolean force);
-    void postReplay();
+    default void update(Command prev, Command updated, boolean force) {}
+    default void postReplay() {}
+    default void prune(TxnId syncId, Ranges ranges, RedundantBefore redundantBefore) {}
+    default void save(File file) throws IOException {}
+    default Object load(File file) throws IOException { return null; }
+    default void restore(Object loaded) {}
 }

src/java/org/apache/cassandra/service/accord/TokenRange.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import com.google.common.annotations.VisibleForTesting;
 
+import accord.api.ProtocolModifiers.RangeSpec;
 import accord.api.RoutingKey;
 import accord.primitives.Range;
 import accord.utils.Invariants;
@@ -37,9 +38,13 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.TokenKey;
 import org.apache.cassandra.utils.ObjectSizes;
 
-public class TokenRange extends Range.EndInclusive
+public class TokenRange extends Range
 {
     public static final long EMPTY_SIZE = ObjectSizes.measure(new TokenRange(TokenKey.min(TableId.fromLong(0), Murmur3Partitioner.instance), TokenKey.max(TableId.fromLong(0), Murmur3Partitioner.instance)));
+    static
+    {
+        Invariants.require(RangeSpec.isEndInclusive());
+    }
 
     // Don't make this public use create or createUnsafe
     protected TokenRange(TokenKey start, TokenKey end)
@@ -124,6 +129,14 @@ public class TokenRange extends Range.EndInclusive
         return new org.apache.cassandra.dht.Range<>(left, right);
     }
 
+    public static TokenRange fromKeyspaceRange(TableId tableId, org.apache.cassandra.dht.Range<Token> range)
+    {
+        Token left = range.left, right = range.right;
+        TokenKey start = left.isMinimum() ? TokenKey.min(tableId, left.getPartitioner()) : new TokenKey(tableId, left);
+        TokenKey end = right.isMinimum() ? TokenKey.max(tableId, right.getPartitioner()) : new TokenKey(tableId, right);
+        return create(start, end);
+    }
+
     public static final Serializer serializer = new Serializer();
 
     public static final class Serializer implements UnversionedSerializer<TokenRange>

src/java/org/apache/cassandra/service/accord/api/AccordAgent.java

@@ -38,9 +38,11 @@ import accord.api.ReplicaEventListener;
 import accord.api.RoutingKey;
 import accord.api.Tracing;
 import accord.coordinate.Coordination;
+import accord.coordinate.Exhausted;
 import accord.coordinate.Preempted;
 import accord.coordinate.Timeout;
 import accord.local.Command;
+import accord.local.LogUnavailableException;
 import accord.local.Node;
 import accord.local.SafeCommand;
 import accord.local.SafeCommandStore;
@@ -66,6 +68,7 @@ import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import accord.utils.async.Cancellable;
 
+import org.apache.cassandra.config.AccordSpec;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.exceptions.RequestTimeoutException;
@@ -121,6 +124,7 @@ public class AccordAgent implements Agent, OwnershipEventListener
     private final AccordTracing tracing = new AccordTracing();
     private final RandomSource random = new DefaultRandom();
     protected Node.Id self;
+    protected AccordSpec config;
 
     public AccordAgent()
     {
@@ -143,9 +147,10 @@ public class AccordAgent implements Agent, OwnershipEventListener
         return this;
     }
 
-    public void setNodeId(Node.Id id)
+    public void setup(Node.Id id)
     {
         self = id;
+        config = DatabaseDescriptor.getAccord();
     }
 
     @Override
@@ -186,7 +191,7 @@ public class AccordAgent implements Agent, OwnershipEventListener
         else
         {
             logger.error(message, phase, ranges, ". Retrying in " + retryDelayMicros + "us.", failure);
-            AccordService.instance().scheduler().once(() -> {
+            AccordService.unsafeInstance().scheduler().once(() -> {
                 logger.info("Retrying bootstrap of {}", ranges);
                 retry.run();
             }, retryDelayMicros, MICROSECONDS);
@@ -205,7 +210,7 @@ public class AccordAgent implements Agent, OwnershipEventListener
             return;
 
         AccordSystemMetrics.metrics.errors.inc();
-        if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted)
+        if (t instanceof CancellationException || t instanceof TimeoutException || t instanceof Timeout || t instanceof Preempted || t instanceof Exhausted || t instanceof LogUnavailableException)
             // TODO (required): leaky logger, permitting multiple messages per time period and reporting how many were dropped
             noSpamLogger.warn("", t);
         else
@@ -266,6 +271,21 @@ public class AccordAgent implements Agent, OwnershipEventListener
         return 1024;
     }
 
+    @Override
+    public boolean softReject(long unappliedCount, long maxUnappliedAge, long cumulativeUnappliedAge)
+    {
+        return unappliedCount > config.min_soft_reject_count
+               && (unappliedCount > config.max_soft_reject_count
+                || maxUnappliedAge > config.soft_reject_age.toMicroseconds()
+                || cumulativeUnappliedAge > config.soft_reject_cumulative_age.toMicroseconds());
+    }
+
+    @Override
+    public boolean hardReject(int softRejectCount, int totalCount)
+    {
+        return (softRejectCount / (float) totalCount) >= config.hard_reject_ratio;
+    }
+
     /**
      * Create an empty transaction that Accord can use for its internal transactions. This is not suitable
      * for tests since it skips validation done by regular transactions.
@@ -295,10 +315,19 @@ public class AccordAgent implements Agent, OwnershipEventListener
     public long slowCoordinatorDelay(Node node, SafeCommandStore safeStore, TxnId txnId, TimeUnit units, int attempt)
     {
         SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId);
-        Invariants.nonNull(safeCommand);
+        if (safeCommand == null)
+        {
+            noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException());
+            return recover(txnId).computeWait(attempt, units);
+        }
 
         Command command = safeCommand.current();
-        Invariants.nonNull(command);
+        if (command == null)
+        {
+            noSpamLogger.warn("{} invoked slowCoordinatorDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException());
+            return recover(txnId).computeWait(attempt, units);
+        }
+
 
         // TODO (expected): make this a configurable calculation on normal request latencies (like ContentionStrategy)
         long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
@@ -368,7 +397,20 @@ public class AccordAgent implements Agent, OwnershipEventListener
     @Override
     public long slowReplicaDelay(Node node, SafeCommandStore safeStore, TxnId txnId, int attempt, BlockedUntil blockedUntil, TimeUnit units)
     {
-        Command command = Invariants.nonNull(safeStore.unsafeGetNoCleanup(txnId).current());
+        SafeCommand safeCommand = safeStore.unsafeGetNoCleanup(txnId);
+        if (safeCommand == null)
+        {
+            noSpamLogger.warn("{} invoked slowReplicaDelay for {} without having it in cache", safeStore.commandStore(), txnId, new RuntimeException());
+            return fetch(txnId).computeWait(attempt, units);
+        }
+
+        Command command = safeCommand.current();
+        if (command == null)
+        {
+            noSpamLogger.warn("{} invoked slowReplicaDelay for {} without knowing the command", safeStore.commandStore(), txnId, new RuntimeException());
+            return fetch(txnId).computeWait(attempt, units);
+        }
+
         long nowMicros = MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
         long mostRecentStart = mostRecentStart(command, nowMicros);
         long waitMicros = fetch(txnId).computeWait(attempt, units);

src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java

@@ -42,7 +42,7 @@ public abstract class AccordRoutableKey implements RoutableKey
         // -1 means dynamic
         int fixedKeyLengthForPrefix(Object prefix);
         int serializedSizeOfPrefix(Object prefix);
-        int serializedSizeWithoutPrefix(K key);
+        int serializedSizeWithoutPrefixOrLength(K key);
         void serializePrefix(Object prefix, DataOutputPlus out) throws IOException;
         void serializeWithoutPrefixOrLength(K key, DataOutputPlus out) throws IOException;
         Object deserializePrefix(DataInputPlus in) throws IOException;

src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java

@@ -35,11 +35,11 @@ import org.apache.cassandra.locator.DynamicEndpointSnitch;
 import org.apache.cassandra.locator.Endpoint;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.NodeProximity;
-import org.apache.cassandra.service.accord.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Sortable;
 
-import static org.apache.cassandra.service.accord.AccordEndpointMapper.NodeStatus.HEALTHY;
+import static org.apache.cassandra.service.accord.topology.AccordEndpointMapper.NodeStatus.HEALTHY;
 
 public class AccordTopologySorter implements TopologySorter
 {

src/java/org/apache/cassandra/service/accord/api/TokenKey.java

@@ -375,7 +375,7 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
 
         public ByteBuffer serialize(TokenKey key)
         {
-            int size = key.table.serializedCompactComparableSize() + serializedSizeWithoutPrefix(key);
+            int size = key.table.serializedCompactComparableSize() + serializedSizeWithoutPrefixOrLength(key);
             ByteBuffer result = ByteBuffer.allocate(size);
             result.position(key.table.serializeCompactComparable(result, ByteBufferAccessor.instance, 0));
             serializeWithoutPrefixOrLength(key, result);
@@ -417,7 +417,7 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
         }
 
         @Override
-        public int serializedSizeWithoutPrefix(TokenKey key)
+        public int serializedSizeWithoutPrefixOrLength(TokenKey key)
         {
             return 2 + key.token.getPartitioner().accordSerializedSize(key.token);
         }
@@ -445,7 +445,7 @@ public final class TokenKey extends AccordRoutableKey implements RoutingKey, Ran
         public ByteBuffer serializeWithoutPrefixOrLength(TokenKey key)
         {
             IPartitioner partitioner = key.token.getPartitioner();
-            ByteBuffer result = ByteBuffer.allocate(serializedSizeWithoutPrefix(key));
+            ByteBuffer result = ByteBuffer.allocate(serializedSizeWithoutPrefixOrLength(key));
             serializeWithoutPrefixOrLength(key, result, partitioner);
             result.flip();
             return result;

src/java/org/apache/cassandra/service/accord/debug/DebugTxnGraph.java

@@ -22,10 +22,12 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Queue;
+import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
@@ -171,7 +173,7 @@ public abstract class DebugTxnGraph<T, P>
 
     protected AsyncChain<TxnInfos<T>> visitRoot(SafeCommandStore safeStore, Command command, P param)
     {
-        return visitParent(safeStore, command, param, new HashMap<>(), 0);
+        return visitParent(safeStore, command, param, new HashMap<>(), new HashSet<>(), 0);
     }
 
     void visit(long deadlineNanos) throws TimeoutException
@@ -227,17 +229,18 @@ public abstract class DebugTxnGraph<T, P>
         }).flatMap(i -> i);
     }
 
-    private AsyncChain<TxnInfos<T>> submitParent(CommandStore commandStore, TxnId txnId, P param, Map<TxnId, SaveInfo> infos, int depth)
+    private AsyncChain<TxnInfos<T>> submitParent(CommandStore commandStore, TxnId txnId, P param, Map<TxnId, SaveInfo> infos, Set<TxnId> visitedParent, int depth)
     {
+
         return commandStore.chain(PreLoadContext.contextFor(txnId, "Populate txn_graph"), safeStore -> {
             Command command = safeStore.unsafeGetNoCleanup(txnId).current();
             if (command == null || command.saveStatus() == SaveStatus.Uninitialised)
                 return AsyncChains.<TxnInfos<T>>success(null);
-            return visitParent(safeStore, command, param, infos, depth);
+            return visitParent(safeStore, command, param, infos, visitedParent, depth);
         }).flatMap(i -> i);
     }
 
-    private AsyncChain<TxnInfos<T>> visitParent(SafeCommandStore safeStore, Command command, P param, Map<TxnId, SaveInfo> infos, int depth)
+    private AsyncChain<TxnInfos<T>> visitParent(SafeCommandStore safeStore, Command command, P param, Map<TxnId, SaveInfo> infos, Set<TxnId> visitedParent, int depth)
     {
         CommandStore commandStore = safeStore.commandStore();
         if (depth < maxDepth)
@@ -296,7 +299,8 @@ public abstract class DebugTxnGraph<T, P>
                                 if (!next.saveStatus.hasBeen(Status.Committed) || next.saveStatus.hasBeen(Status.Truncated))
                                     return;
 
-                                queued.add(submitParent(commandStore, next.txnId, param, infos, depth + 1));
+                                if (visitedParent.add(next.txnId))
+                                    queued.add(submitParent(commandStore, next.txnId, param, infos, visitedParent, depth + 1));
                             });
                             callback.accept(build(commandStore, depth, command, list, intersecting, param), null);
                             return null;

src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java

@@ -43,7 +43,7 @@ import accord.primitives.Writes;
 import accord.topology.Topologies;
 
 import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.service.accord.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 

src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java

@@ -79,10 +79,10 @@ import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageProxy;
-import org.apache.cassandra.service.accord.AccordEndpointMapper;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.TokenKey;
 import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;

src/java/org/apache/cassandra/service/accord/journal/AbstractSegmentCompactor.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -46,8 +46,8 @@ import org.apache.cassandra.journal.SegmentCompactor;
 import org.apache.cassandra.journal.StaticSegment;
 import org.apache.cassandra.journal.StaticSegment.KeyOrderReader;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightImage;
-import org.apache.cassandra.service.accord.AccordJournalValueSerializers.FlyweightSerializer;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.serializers.Version;
 import org.apache.cassandra.utils.BulkIterator;
 import org.apache.cassandra.utils.NoSpamLogger;
@@ -59,9 +59,9 @@ import static java.util.concurrent.TimeUnit.MINUTES;
 /**
  * Segment compactor: takes static segments and compacts them into a single SSTable.
  */
-public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V>
+public abstract class AbstractSegmentCompactor<V> implements SegmentCompactor<JournalKey, V>
 {
-    protected static final Logger logger = LoggerFactory.getLogger(AbstractAccordSegmentCompactor.class);
+    protected static final Logger logger = LoggerFactory.getLogger(AbstractSegmentCompactor.class);
     private static final NoSpamLogger.NoSpamLogStatement unknownTable = NoSpamLogger.getStatement(logger, "Unknown (probably dropped) TableId {} reading {}; skipping record", 1L, MINUTES);
 
     static final Object[] rowTemplate = BTree.build(BulkIterator.of(new Object[2]), 2, UpdateFunction.noOp);
@@ -71,7 +71,7 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
     protected final ColumnFamilyStore cfs;
     protected final long timestamp = ClientState.getTimestamp();
 
-    public AbstractAccordSegmentCompactor(Version userVersion, ColumnFamilyStore cfs)
+    public AbstractSegmentCompactor(Version userVersion, ColumnFamilyStore cfs)
     {
         this.userVersion = userVersion;
         this.userVersionCell = BufferCell.live(AccordKeyspace.JournalColumns.user_version, timestamp, Int32Type.instance.decompose(userVersion.version));
@@ -120,8 +120,8 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
         initializeWriter(estimatedKeyCount);
 
         JournalKey key = null;
-        FlyweightImage builder = null;
-        FlyweightSerializer<Object, FlyweightImage> serializer = null;
+        Merger merger = null;
+        MergeSerializer serializer = null;
         long firstDescriptor = -1, lastDescriptor = -1;
         int firstOffset = -1, lastOffset = -1;
         try
@@ -131,12 +131,12 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
             {
                 if (key == null || !reader.key().equals(key))
                 {
-                    maybeWritePartition(key, builder, serializer, firstDescriptor, firstOffset);
+                    maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset);
                     switchPartitions();
                     key = reader.key();
-                    serializer = (FlyweightSerializer<Object, FlyweightImage>) key.type.serializer;
-                    builder = serializer.mergerFor();
-                    builder.reset(key);
+                    serializer = key.type.serializer;
+                    merger = serializer.mergerFor();
+                    merger.reset(key);
                     firstDescriptor = lastDescriptor = -1;
                     firstOffset = lastOffset = -1;
                 }
@@ -146,10 +146,10 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
                 boolean advanced;
                 do
                 {
-                    if (builder == null)
+                    if (merger == null)
                     {
-                        builder = serializer.mergerFor();
-                        builder.reset(key);
+                        merger = serializer.mergerFor();
+                        merger.reset(key);
                     }
 
                     try (DataInputBuffer in = new DataInputBuffer(reader.record(), false))
@@ -162,7 +162,7 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
                                                reader.offset() < lastOffset,
                                                "Offsets were accessed out of order: %d was accessed after %s", reader.offset(), lastOffset);
                         }
-                        serializer.deserialize(key, builder, in, realVersion);
+                        serializer.deserialize(key, merger, in, realVersion);
                         lastDescriptor = reader.descriptor.timestamp;
                         lastOffset = reader.offset();
                         if (firstDescriptor == -1)
@@ -174,8 +174,8 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
 
                     if (considerWritingKey())
                     {
-                        maybeWritePartition(key, builder, serializer, firstDescriptor, firstOffset);
-                        builder = null;
+                        maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset);
+                        merger = null;
                         firstDescriptor = lastDescriptor = -1;
                         firstOffset = lastOffset = -1;
                     }
@@ -186,7 +186,7 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
                 else reader.close();
             }
 
-            maybeWritePartition(key, builder, serializer, firstDescriptor, firstOffset);
+            maybeWritePartition(key, merger, serializer, firstDescriptor, firstOffset);
             switchPartitions();
         }
         catch (UnknownTableException e)
@@ -208,9 +208,9 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
         return Collections.emptyList();
     }
 
-    private void maybeWritePartition(JournalKey key, FlyweightImage builder, FlyweightSerializer<Object, FlyweightImage> serializer, long descriptor, int offset) throws IOException
+    private void maybeWritePartition(JournalKey key, Merger merger, MergeSerializer<Object, ? super Merger, Merger> serializer, long descriptor, int offset) throws IOException
     {
-        if (builder != null)
+        if (merger != null)
         {
             DecoratedKey decoratedKey = AccordKeyspace.JournalColumns.decorate(key);
             Invariants.requireArgument(prevKey == null || normalize(decoratedKey.compareTo(prevDecoratedKey)) == normalize(JournalKey.SUPPORT.compare(key, prevKey)),
@@ -222,7 +222,7 @@ public abstract class AbstractAccordSegmentCompactor<V> implements SegmentCompac
             Object[] rowData = rowTemplate.clone();
             try (DataOutputBuffer out = DataOutputBuffer.scratchBuffer.get())
             {
-                serializer.reserialize(key, builder, out, userVersion);
+                serializer.reserialize(key, merger, out, userVersion);
                 rowData[0] = BufferCell.live(AccordKeyspace.JournalColumns.record, timestamp, out.asNewBuffer());
             }
             rowData[1] = userVersionCell;

src/java/org/apache/cassandra/service/accord/journal/AccordJournal.java

@@ -0,0 +1,698 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.agrona.collections.Long2LongHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.impl.CommandChange;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.CommandStores;
+import accord.local.CommandStores.RangesForEpoch;
+import accord.local.DurableBefore;
+import accord.local.Node;
+import accord.local.RedundantBefore;
+import accord.primitives.EpochSupplier;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import accord.utils.PersistentField;
+
+import org.apache.cassandra.config.AccordSpec.JournalSpec;
+import org.apache.cassandra.config.AccordSpec.JournalSpec.ReplayMode;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.journal.Compactor;
+import org.apache.cassandra.journal.Descriptor;
+import org.apache.cassandra.journal.Journal;
+import org.apache.cassandra.journal.Params;
+import org.apache.cassandra.journal.RecordConsumer;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.journal.Segments;
+import org.apache.cassandra.journal.ValueSerializer;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.journal.Merger.KeepFirst;
+import org.apache.cassandra.service.accord.journal.RangeSearcher.NoopJournalRangeSearcher;
+import org.apache.cassandra.service.accord.serializers.Version;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static accord.api.Journal.Load.MINIMAL;
+import static accord.api.Journal.Load.MINIMAL_WITH_DEPS;
+import static accord.local.Cleanup.Input.FULL;
+import static org.apache.cassandra.config.AccordSpec.RangeIndexMode.journal_sai;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccord;
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordJournalDirectory;
+import static org.apache.cassandra.service.accord.JournalKey.Type.COMMAND_DIFF;
+import static org.apache.cassandra.service.accord.journal.ReplayMarkers.safeStopMarker;
+import static org.apache.cassandra.service.accord.journal.ReplayMarkers.startMarker;
+import static org.apache.cassandra.service.accord.journal.ReplayMarkers.writeMarker;
+import static org.apache.cassandra.service.accord.journal.TopologyRecord.newTopology;
+
+public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
+{
+    private static final Logger logger = LoggerFactory.getLogger(AccordJournal.class);
+
+    @VisibleForTesting
+    protected final Journal<JournalKey, Object> segments;
+    protected final ColumnFamilyStore table;
+    @VisibleForTesting
+    protected final @Nullable RangeSearchManager rangeSearch;
+    protected final OpOrder readOrder;
+    private final Params params;
+
+    public AccordJournal(Params params)
+    {
+        this(params, new File(getAccordJournalDirectory()), Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL));
+    }
+
+    @VisibleForTesting
+    public AccordJournal(Params params, File directory, ColumnFamilyStore table)
+    {
+        Version userVersion = Version.fromVersion(params.userVersion());
+        this.rangeSearch = RangeSearchManager.ifEnabled(table);
+        this.table = table;
+        this.readOrder = table.readOrdering;
+        this.params = params;
+        // initialise journal last because we call a self method to initialise its compactor
+        this.segments = new Journal<>("AccordJournal", directory, params, JournalKey.SUPPORT,
+                                      new ValueSerializer.Unsupported<>(),
+                                      compactor(table, userVersion),
+                                      table.readOrdering);
+    }
+
+    @Override
+    public void open(Node node)
+    {
+        segments.open();
+    }
+
+    public void start(Node node)
+    {
+        if (rangeSearch != null)
+            rangeSearch.start();
+
+        long maxTableDescriptor = maxTableDescriptor();
+        segments.start(maxTableDescriptor);
+    }
+
+    public Descriptor stop()
+    {
+        return segments.stop();
+    }
+
+    public void close()
+    {
+        segments.close();
+    }
+
+    public boolean awaitTerminationUntil(long deadlineNanos) throws InterruptedException
+    {
+        try
+        {
+            segments.awaitTerminationUntil(deadlineNanos);
+            return true;
+        }
+        catch (TimeoutException e)
+        {
+            return false;
+        }
+    }
+
+    @Override
+    public void saveCommand(int commandStoreId, CommandUpdate update, @Nullable Runnable onFlush)
+    {
+        CommandChangeWriter change = CommandChangeWriter.make(update.before, update.after);
+        if (change == null)
+        {
+            if (onFlush != null)
+                onFlush.run();
+            return;
+        }
+
+        JournalKey key = new JournalKey(update.txnId, COMMAND_DIFF, commandStoreId);
+        RecordPointer pointer = segments.asyncWrite(key, change);
+        if (rangeSearch != null)
+            onFlush = merge(onFlush, rangeSearch.maybeIndex(key, pointer, change));
+        if (onFlush != null)
+            segments.onDurable(pointer, onFlush);
+    }
+
+    <T> void append(JournalKey key, T write, Runnable onFlush)
+    {
+        RecordPointer pointer = appendInternal(key, write);
+        if (onFlush != null)
+            segments.onDurable(pointer, onFlush);
+    }
+
+    private <T> RecordPointer appendInternal(JournalKey key, T write)
+    {
+        MergeSerializer<T, ?, ?> serializer = (MergeSerializer<T, ?, ?>) key.type.serializer;
+        return segments.asyncWrite(key, (out, userVersion) -> serializer.serialize(key, write, out, Version.fromVersion(userVersion)));
+    }
+
+
+    public long maxTableDescriptor()
+    {
+        return table.getTracker().getView().liveSSTables()
+                    .stream()
+                    .filter(sst -> sst.getSSTableMetadata().totalRows > 0)
+                    .map(sst -> LongType.instance.compose(sst.getSSTableMetadata().coveredClustering.end().bufferAt(0)))
+                    .max(Long::compare).orElse(0L);
+    }
+
+    public long maxDescriptor()
+    {
+        return Math.max(segments.maxDescriptor(), maxTableDescriptor());
+    }
+
+    public Params configuration()
+    {
+        return params;
+    }
+
+    @Override
+    public Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
+    {
+        CommandChanges builder = load(commandStoreId, txnId);
+        builder.maybeCleanup(true, FULL, redundantBefore, durableBefore);
+        return builder.construct(redundantBefore);
+    }
+
+    @Override
+    public List<DebugEntry> debugCommand(int commandStoreId, TxnId txnId)
+    {
+        JournalKey key = new JournalKey(txnId, COMMAND_DIFF, commandStoreId);
+        List<DebugEntry> result = new ArrayList<>();
+        readAll(key, (long segment, int position, JournalKey k, ByteBuffer buffer, int userVersion) -> {
+            CommandChanges builder = new CommandChanges(txnId);
+            new RecordConsumerAdapter<>(builder::deserializeNext).accept(segment, position, k, buffer, userVersion);
+            result.add(new DebugEntry(segment, position, builder));
+        });
+        return result;
+    }
+
+    @Override
+    public Command.Minimal loadMinimal(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
+    {
+        CommandChanges builder = CommandChanges.cleanupAndFilter(loadDiffs(commandStoreId, txnId, MINIMAL), redundantBefore, durableBefore);
+        return builder == null ? null : builder.asMinimal();
+    }
+
+    @Override
+    public Command.MinimalWithDeps loadMinimalWithDeps(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
+    {
+        CommandChanges builder = CommandChanges.cleanupAndFilter(loadDiffs(commandStoreId, txnId, MINIMAL_WITH_DEPS), redundantBefore, durableBefore);
+        return builder == null ? null : builder.asMinimalWithDeps();
+    }
+
+    private CommandChanges loadDiffs(int commandStoreId, TxnId txnId, Load load)
+    {
+        JournalKey key = new JournalKey(txnId, COMMAND_DIFF, commandStoreId);
+        CommandChanges builder = new CommandChanges(txnId, load);
+        readAll(key, builder::deserializeNext);
+        return builder;
+    }
+
+    @VisibleForTesting
+    public CommandChanges load(int commandStoreId, TxnId txnId)
+    {
+        return loadDiffs(commandStoreId, txnId, Load.ALL);
+    }
+
+    @Override
+    public List<TopologyUpdate> loadTopologies()
+    {
+        List<accord.api.Journal.TopologyUpdate> images = new ArrayList<>();
+        try (CloseableIterator<accord.api.Journal.TopologyUpdate> iter = new CloseableIterator<>()
+        {
+            final CloseableIterator<Journal.KeyRefs<JournalKey>> iter = keyIterator(TopologyRecord.journalKey(0L),
+                                                                                    TopologyRecord.journalKey(Timestamp.MAX_EPOCH),
+                                                                                    true, 0);
+            TopologyRecord.TopologyImage prev = null;
+
+            @Override
+            public boolean hasNext()
+            {
+                return iter.hasNext();
+            }
+
+            @Override
+            public accord.api.Journal.TopologyUpdate next()
+            {
+                Journal.KeyRefs<JournalKey> ref = iter.next();
+                MergeSerializers.TopologyMerger reader = readAll(ref.key());
+                if (reader.read().kind() == TopologyRecord.Kind.Repeat)
+                {
+                    if (prev == null)
+                    {
+                        logger.error("Encountered TopologyImage Repeat record for epoch {}, but no prior image record was found", ref.key().id.epoch());
+                        return null;
+                    }
+                    prev = reader.read().asImage(Invariants.nonNull(prev.getUpdate()));
+                }
+                else prev = reader.read();
+
+                return new accord.api.Journal.TopologyUpdate(prev.getUpdate().commandStores,
+                                                             prev.getUpdate().global);
+            }
+
+            @Override
+            public void close()
+            {
+                iter.close();
+            }
+        })
+        {
+            accord.api.Journal.TopologyUpdate prev = null;
+            while (iter.hasNext())
+            {
+                accord.api.Journal.TopologyUpdate next = iter.next();
+                if (next == null)
+                    continue;
+
+                Invariants.require(prev == null || next.global.epoch() > prev.global.epoch());
+                // Due to partial compaction, we can clean up only some of the old epochs, creating gaps. We skip these epochs here.
+                if (prev != null && next.global.epoch() > prev.global.epoch() + 1)
+                    images.clear();
+
+                images.add(next);
+                prev = next;
+            }
+        }
+        return images;
+    }
+
+    @Override
+    public void saveTopology(TopologyUpdate topologyUpdate, Runnable onFlush)
+    {
+        append(TopologyRecord.journalKey(topologyUpdate.global.epoch()),
+               newTopology(topologyUpdate),
+               onFlush);
+    }
+
+    @Override
+    public RedundantBefore loadRedundantBefore(int commandStoreId)
+    {
+        KeepFirst<RedundantBefore> accumulator = readLast(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, commandStoreId));
+        return accumulator.get();
+    }
+
+    @Override
+    public NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int commandStoreId)
+    {
+        KeepFirst<NavigableMap<TxnId, Ranges>> accumulator = readLast(new JournalKey(TxnId.NONE, JournalKey.Type.BOOTSTRAP_BEGAN_AT, commandStoreId));
+        return accumulator.get();
+    }
+
+    @Override
+    public NavigableMap<Timestamp, Ranges> loadSafeToRead(int commandStoreId)
+    {
+        KeepFirst<NavigableMap<Timestamp, Ranges>> accumulator = readLast(new JournalKey(TxnId.NONE, JournalKey.Type.SAFE_TO_READ, commandStoreId));
+        return accumulator.get();
+    }
+
+    @Override
+    public CommandStores.RangesForEpoch loadRangesForEpoch(int commandStoreId)
+    {
+        KeepFirst<RangesForEpoch> accumulator = readLast(new JournalKey(TxnId.NONE, JournalKey.Type.RANGES_FOR_EPOCH, commandStoreId));
+        return accumulator.get();
+    }
+
+    @Override
+    public PersistentField.Persister<DurableBefore, DurableBefore> durableBeforePersister()
+    {
+        return new DurableBeforePersister(this);
+    }
+
+    @Override
+    public void saveStoreState(int commandStoreId, FieldUpdates fieldUpdates, Runnable onFlush)
+    {
+        RecordPointer pointer = null;
+        if (fieldUpdates.newRedundantBefore != null)
+            pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, commandStoreId), fieldUpdates.newRedundantBefore);
+        if (fieldUpdates.newBootstrapBeganAt != null)
+            pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.BOOTSTRAP_BEGAN_AT, commandStoreId), fieldUpdates.newBootstrapBeganAt);
+        if (fieldUpdates.newSafeToRead != null)
+            pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.SAFE_TO_READ, commandStoreId), fieldUpdates.newSafeToRead);
+        if (fieldUpdates.newRangesForEpoch != null)
+            pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.RANGES_FOR_EPOCH, commandStoreId), fieldUpdates.newRangesForEpoch);
+
+        if (onFlush == null)
+            return;
+
+        if (pointer != null)
+            segments.onDurable(pointer, onFlush);
+        else
+            onFlush.run();
+    }
+
+    public <BUILDER extends Merger> BUILDER readAll(JournalKey key)
+    {
+        Invariants.require(segments.isReadable());
+        BUILDER builder = (BUILDER) key.type.serializer.mergerFor();
+        builder.reset(key);
+        // TODO (expected): this can be further improved to avoid allocating lambdas
+        MergeSerializer<?, ? super BUILDER, ? extends BUILDER> serializer = (MergeSerializer<?, ? super BUILDER, ? extends BUILDER>) key.type.serializer;
+        // TODO (expected): for those where we store an image, read only the first entry we find in DESC order
+        readAll(key, (in, userVersion) -> serializer.deserialize(key, builder, in, userVersion));
+        return builder;
+    }
+
+    public <BUILDER extends Merger> BUILDER readLast(JournalKey key)
+    {
+        Invariants.require(segments.isReadable());
+        BUILDER builder = (BUILDER) key.type.serializer.mergerFor();
+        builder.reset(key);
+        // TODO (expected): this can be further improved to avoid allocating lambdas
+        MergeSerializer<?, ? super BUILDER, ? extends BUILDER> serializer = (MergeSerializer<?, ? super BUILDER, ? extends BUILDER>) key.type.serializer;
+        readLast(key, (in, userVersion) -> serializer.deserialize(key, builder, in, userVersion));
+        return builder;
+    }
+
+    public void forEachEntry(JournalKey key, Reader reader)
+    {
+        readAll(key, reader);
+    }
+
+    public interface Reader
+    {
+        void read(DataInputPlus input, Version userVersion) throws IOException;
+
+        default void read(ByteBuffer buffer, Version userVersion)
+        {
+            try (DataInputBuffer in = new DataInputBuffer(buffer, false))
+            {
+                read(in, userVersion);
+            }
+            catch (IOException e)
+            {
+                // can only throw if serializer is buggy or bytes got corrupted
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+
+    static class RecordConsumerAdapter<K> implements RecordConsumer<K>
+    {
+        protected final Reader reader;
+        private long prevSegment = Long.MAX_VALUE;
+        private long prevPosition = Long.MAX_VALUE;
+
+        RecordConsumerAdapter(Reader reader)
+        {
+            this.reader = reader;
+        }
+
+        @Override
+        public void accept(long segment, int position, K key, ByteBuffer buffer, int userVersion)
+        {
+            Invariants.require(segment <= prevSegment,
+                               "Records should always be iterated over in a reverse order, but segment %d was seen after %d while reading %s", segment, prevSegment, key);
+            Invariants.require(segment != prevSegment || position < prevPosition,
+                               "Records should always be iterated over in a reverse order, but position %d was seen after %d for segment %d while reading %s", position, prevPosition, segment, key);
+            reader.read(buffer, Version.fromVersion(userVersion));
+            prevSegment = segment;
+            prevPosition = position;
+        }
+    }
+
+    /**
+     * Perform a read from Journal table, followed by the reads from all journal segments.
+     * <p>
+     * When reading from journal segments, skip descriptors that were read from the table.
+     */
+    public void readAll(JournalKey key, Reader reader)
+    {
+        readAll(key, new RecordConsumerAdapter<>(reader));
+    }
+
+    public void readAll(JournalKey key, RecordConsumer<JournalKey> reader)
+    {
+        try (OpOrder.Group readOrder = table.readOrdering.start())
+        {
+            // SELECT segments first, to avoid missing segments due to races compacting segment->sstable
+            Segments<JournalKey, Object> segments = this.segments.segments();
+            try (TableRecordIterator table = TableRecordIterator.all(this.table, key, readOrder))
+            {
+                boolean hasTableData = table.advance();
+                long minSegment = hasTableData ? table.segment : Long.MIN_VALUE;
+                // First, read all journal entries newer than anything flushed into sstables
+                Journal.readAll(key, (segment, position, key1, buffer, userVersion) -> {
+                    if (segment > minSegment)
+                        reader.accept(segment, position, key1, buffer, userVersion);
+                }, readOrder, segments);
+
+                // Then, read SSTables
+                while (hasTableData)
+                {
+                    reader.accept(table.segment, table.offset, key, table.value, table.userVersion);
+                    hasTableData = table.advance();
+                }
+            }
+        }
+    }
+
+    public void readLast(JournalKey key, Reader reader)
+    {
+        readLast(key, new RecordConsumerAdapter<>(reader));
+    }
+
+    public void readLast(JournalKey key, RecordConsumer<JournalKey> reader)
+    {
+        try (OpOrder.Group readOrder = table.readOrdering.start())
+        {
+            Segments<JournalKey, Object> segments = this.segments.segments();
+            try (TableRecordIterator table = TableRecordIterator.all(this.table, key, readOrder))
+            {
+                boolean hasTableData = table.advance();
+                long minSegment = hasTableData ? table.segment : Long.MIN_VALUE;
+
+                class JournalReader implements RecordConsumer<JournalKey>
+                {
+                    boolean read;
+                    @Override
+                    public void accept(long segment, int position, JournalKey key, ByteBuffer buffer, int userVersion)
+                    {
+                        if (segment > minSegment)
+                        {
+                            reader.accept(segment, position, key, buffer, userVersion);
+                            read = true;
+                        }
+                    }
+                }
+
+                // First, read all journal entries newer than anything flushed into sstables
+                JournalReader journalReader = new JournalReader();
+                Journal.readLast(key, journalReader, readOrder, segments);
+
+                // Then, read SSTables, if we haven't found a record already
+                if (hasTableData && !journalReader.read)
+                    reader.accept(table.segment, table.offset, key, table.value, table.userVersion);
+            }
+        }
+    }
+
+    public CloseableIterator<Journal.KeyRefs<JournalKey>> keyIterator(@Nullable JournalKey min, @Nullable JournalKey max, boolean includeActive, long minSegment)
+    {
+        // the readOrder is taken only to ensure we get a consistent snapshot of both segments and sstables;
+        // we take references to the segments and sstables directly, and do not need to manage the readOrder for the lifetime of the iterator
+        try (OpOrder.Group readOrder = this.readOrder.start())
+        {
+            return new TableAndSegmentKeyIterator<>(segments, table, min, max, includeActive, minSegment);
+        }
+    }
+
+    public void forEach(Consumer<JournalKey> consumer, boolean includeActive, long minSegment)
+    {
+        forEach(consumer, null, null, includeActive, minSegment);
+    }
+
+    public void forEach(Consumer<JournalKey> consumer, @Nullable JournalKey min, @Nullable JournalKey max, boolean includeActive, long minSegment)
+    {
+        try (CloseableIterator<Journal.KeyRefs<JournalKey>> iter = keyIterator(min, max, includeActive, minSegment))
+        {
+            while (iter.hasNext())
+            {
+                Journal.KeyRefs<JournalKey> ref = iter.next();
+                consumer.accept(ref.key());
+            }
+        }
+    }
+
+    public Compactor<JournalKey, Object> compactor()
+    {
+        return segments.compactor();
+    }
+
+    protected org.apache.cassandra.journal.SegmentCompactor<JournalKey, Object> compactor(ColumnFamilyStore cfs, Version userVersion)
+    {
+        if (rangeSearch == null)
+        {
+            Invariants.require(getAccord().range_index_mode != journal_sai, "range_index_mode is journal_sai, but the storage attached index was not found on initialisation");
+            return new SegmentCompactor<>(userVersion, cfs);
+        }
+
+        return rangeSearch.compactor(cfs, userVersion);
+    }
+
+    public void forceCompaction()
+    {
+        table.forceMajorCompaction();
+    }
+
+    @Override
+    public void purge(CommandStores commandStores, EpochSupplier minEpoch)
+    {
+        segments.closeCurrentSegmentForTestingIfNonEmpty();
+        segments.runCompactorForTesting();
+        forceCompaction();
+    }
+
+    public void replay(CommandStore commandStore, ReplayMode replayMode, long minSegmentId)
+    {
+        Long2LongHashMap minSegments = new Long2LongHashMap(0);
+        minSegments.put(commandStore.id(), minSegmentId);
+        Replay.replay(this, replayMode, new CommandStore[] {commandStore }, minSegments);
+    }
+
+    @Override
+    public boolean replay(CommandStores commandStores, Object param)
+    {
+        ReplayMode mode = params instanceof JournalSpec ? ((JournalSpec)params).replay
+                                                        : getAccord().journal.replay;
+        return Replay.replay(this, mode, commandStores.all(), param);
+    }
+
+    @Override
+    public RangeSearcher rangeSearcher()
+    {
+        if (rangeSearch == null)
+            return NoopJournalRangeSearcher.instance;
+        return rangeSearch.rangeSearcher();
+    }
+
+    public void writeStartMarker()
+    {
+        writeMarker(startMarker(), segments.peekSegmentId());
+    }
+
+    public void writeSafeStopMarker()
+    {
+        segments.fsync();
+        writeMarker(safeStopMarker(), segments.peekSegmentId());
+    }
+
+    private static Runnable merge(Runnable first, Runnable second)
+    {
+        if (first == null) return second;
+        if (second == null) return first;
+        return () ->
+        {
+            try { first.run(); }
+            finally { second.run(); }
+        };
+    }
+
+    public static class DebugEntry implements Supplier<CommandChange.Builder>
+    {
+        public final long segment;
+        public final int position;
+        public final CommandChanges builder;
+
+        public DebugEntry(long segment, int position, CommandChanges builder)
+        {
+            this.segment = segment;
+            this.position = position;
+            this.builder = builder;
+        }
+
+        @Override
+        public CommandChange.Builder get()
+        {
+            return builder;
+        }
+    }
+
+    @VisibleForTesting
+    public Journal<JournalKey, Object> unsafeGetJournal()
+    {
+        return segments;
+    }
+
+    @VisibleForTesting
+    public void closeCurrentSegmentForTestingIfNonEmpty()
+    {
+        segments.closeCurrentSegmentForTestingIfNonEmpty();
+    }
+
+    public void sanityCheck(int commandStoreId, RedundantBefore redundantBefore, Command orig)
+    {
+        CommandChanges builder = load(commandStoreId, orig.txnId());
+        builder.forceResult(orig.result());
+        // We can only use strict equality if we supply result.
+        Command reconstructed = builder.construct(redundantBefore);
+        Invariants.require(orig.equals(reconstructed),
+                           '\n' +
+                           "Original:      %s\n" +
+                           "Reconstructed: %s\n" +
+                           "Diffs:         %s", orig, reconstructed, builder);
+    }
+
+    @VisibleForTesting
+    public void truncateForTesting()
+    {
+        segments.truncateForTesting();
+        if (rangeSearch != null)
+            rangeSearch.safeNotify(SegmentRangeSearcher::truncateForTesting);
+    }
+
+    @VisibleForTesting
+    public void runCompactorForTesting()
+    {
+        segments.runCompactorForTesting();
+    }
+
+    @VisibleForTesting
+    public int inMemorySize()
+    {
+        return segments.currentActiveSegment().index().size();
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/CommandChangeWriter.java

@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import javax.annotation.Nullable;
+
+import accord.impl.CommandChange;
+import accord.local.Cleanup;
+import accord.local.Command;
+import accord.primitives.SaveStatus;
+import accord.utils.Invariants;
+import accord.utils.UnhandledEnum;
+
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.journal.Journal;
+import org.apache.cassandra.service.accord.serializers.CommandSerializers;
+import org.apache.cassandra.service.accord.serializers.DepsSerializers;
+import org.apache.cassandra.service.accord.serializers.ResultSerializers;
+import org.apache.cassandra.service.accord.serializers.Version;
+import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
+
+import static accord.impl.CommandChange.anyFieldChanged;
+import static accord.impl.CommandChange.describeFlags;
+import static accord.impl.CommandChange.getFlags;
+import static accord.impl.CommandChange.isNull;
+import static accord.impl.CommandChange.nextSetField;
+import static accord.impl.CommandChange.toIterableSetFields;
+import static accord.impl.CommandChange.unsetIterable;
+import static accord.impl.CommandChange.validateFlags;
+
+public class CommandChangeWriter implements Journal.Writer
+{
+    final Command after;
+    final int flags;
+
+    private CommandChangeWriter(Command after, int flags)
+    {
+        this.after = after;
+        this.flags = flags;
+    }
+
+    public static CommandChangeWriter make(Command before, Command after)
+    {
+        if (before == after
+            || after == null
+            || after.saveStatus() == SaveStatus.Uninitialised)
+            return null;
+
+        int flags = validateFlags(getFlags(before, after));
+        if (!anyFieldChanged(flags))
+            return null;
+
+        return new CommandChangeWriter(after, flags);
+    }
+
+    @Override
+    public void write(DataOutputPlus out, int userVersion) throws IOException
+    {
+        write(out, Version.fromVersion(userVersion));
+    }
+
+    public void write(DataOutputPlus out, Version userVersion) throws IOException
+    {
+        serialize(after, flags, out, userVersion);
+    }
+
+    private static void serialize(Command command, int flags, DataOutputPlus out, Version userVersion) throws IOException
+    {
+        Invariants.require(flags != 0);
+        out.writeInt(flags);
+
+        int iterable = toIterableSetFields(flags);
+        while (iterable != 0)
+        {
+            CommandChange.Field field = nextSetField(iterable);
+            if (isNull(field, flags))
+            {
+                iterable = unsetIterable(field, iterable);
+                continue;
+            }
+
+            switch (field)
+            {
+                case EXECUTE_AT:
+                    CommandSerializers.ExecuteAtSerializer.serialize(command.txnId(), command.executeAt(), out);
+                    break;
+                case EXECUTES_AT_LEAST:
+                    CommandSerializers.ExecuteAtSerializer.serialize(command.executesAtLeast(), out);
+                    break;
+                case MIN_UNIQUE_HLC:
+                    Invariants.require(command.waitingOn().minUniqueHlc() != 0);
+                    out.writeUnsignedVInt(command.waitingOn().minUniqueHlc());
+                    break;
+                case SAVE_STATUS:
+                    out.writeByte(command.saveStatus().ordinal());
+                    break;
+                case DURABILITY:
+                    out.writeByte(command.durability().encoded());
+                    break;
+                case ACCEPTED:
+                    CommandSerializers.ballot.serialize(command.acceptedOrCommitted(), out);
+                    break;
+                case PROMISED:
+                    CommandSerializers.ballot.serialize(command.promised(), out);
+                    break;
+                case PARTICIPANTS:
+                    CommandSerializers.participants.serialize(command.participants(), out);
+                    break;
+                case PARTIAL_TXN:
+                    CommandSerializers.partialTxn.serialize(command.partialTxn(), out, userVersion);
+                    break;
+                case PARTIAL_DEPS:
+                    DepsSerializers.partialDepsById.serialize(command.partialDeps(), out);
+                    break;
+                case WAITING_ON:
+                    Command.WaitingOn waitingOn = command.waitingOn();
+                    WaitingOnSerializer.serializeBitSetsOnly(command.txnId(), waitingOn, out);
+                    break;
+                case WRITES:
+                    CommandSerializers.writes.serialize(command.writes(), out, userVersion);
+                    break;
+                case RESULT:
+                    ResultSerializers.result.serialize(command.result(), out);
+                    break;
+                case CLEANUP:
+                    Cleanup cleanup;
+                    switch (command.saveStatus())
+                    {
+                        default:
+                            throw new UnhandledEnum(command.saveStatus());
+                        case Erased:
+                            cleanup = Cleanup.ERASE;
+                            break;
+                        case Invalidated:
+                            cleanup = Cleanup.INVALIDATE;
+                            break;
+                    }
+                    out.writeByte(cleanup.ordinal());
+                    break;
+            }
+
+            iterable = unsetIterable(field, iterable);
+        }
+    }
+
+    private boolean hasField(CommandChange.Field fields)
+    {
+        return !isNull(fields, flags);
+    }
+
+    public boolean hasParticipants()
+    {
+        return hasField(CommandChange.Field.PARTICIPANTS);
+    }
+
+    @Override
+    public String toString()
+    {
+        return after.saveStatus() + " " + describeFlags(flags);
+    }
+
+    public static @Nullable ByteBuffer asSerializedChange(Command before, Command after, Version userVersion) throws IOException
+    {
+        // TODO (expected): reusable buffer to build, or pre-size
+        try (DataOutputBuffer out = new DataOutputBuffer())
+        {
+            CommandChangeWriter writer = CommandChangeWriter.make(before, after);
+            if (writer == null)
+                return null;
+
+            writer.write(out, userVersion);
+            return out.asNewBuffer();
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/CommandChanges.java

@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+
+import accord.api.Journal;
+import accord.impl.CommandChange;
+import accord.local.Cleanup;
+import accord.local.DurableBefore;
+import accord.local.RedundantBefore;
+import accord.primitives.PartialDeps;
+import accord.primitives.SaveStatus;
+import accord.primitives.Status;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import accord.utils.UnhandledEnum;
+
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.serializers.CommandSerializers;
+import org.apache.cassandra.service.accord.serializers.DepsSerializers;
+import org.apache.cassandra.service.accord.serializers.ResultSerializers;
+import org.apache.cassandra.service.accord.serializers.Version;
+import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
+
+import static accord.api.Journal.Load.ALL;
+import static accord.impl.CommandChange.Field.CLEANUP;
+import static accord.impl.CommandChange.isChanged;
+import static accord.impl.CommandChange.nextSetField;
+import static accord.impl.CommandChange.toIterableNonNullFields;
+import static accord.impl.CommandChange.toIterableSetFields;
+import static accord.impl.CommandChange.unsetIterable;
+import static accord.impl.CommandChange.validateFlags;
+import static accord.local.Cleanup.Input.FULL;
+
+public class CommandChanges extends CommandChange.Builder implements Merger
+{
+    private final boolean deserializeDeps;
+
+    public CommandChanges()
+    {
+        this(Journal.Load.ALL);
+    }
+
+    public CommandChanges(Journal.Load load)
+    {
+        this(null, load);
+    }
+
+    public CommandChanges(TxnId txnId)
+    {
+        this(txnId, Journal.Load.ALL);
+    }
+
+    public CommandChanges(TxnId txnId, Journal.Load load)
+    {
+        super(txnId, load);
+        deserializeDeps = load == ALL;
+    }
+
+    // applies cleanup and returns null if no command should be returned
+    public static CommandChanges cleanupAndFilter(CommandChanges builder, RedundantBefore redundantBefore, DurableBefore durableBefore)
+    {
+        if (builder.isEmpty())
+            return null;
+
+        Cleanup cleanup = builder.shouldCleanup(FULL, redundantBefore, durableBefore);
+        switch (cleanup)
+        {
+            case VESTIGIAL:
+            case EXPUNGE:
+            case ERASE:
+                return null;
+        }
+        Invariants.require(builder.saveStatus() != null, "No saveSatus loaded, but next was called and cleanup was not: %s", builder);
+        return builder;
+    }
+
+    @Override
+    public PartialDeps partialDeps()
+    {
+        if (partialDeps instanceof ByteBuffer)
+        {
+            try
+            {
+                partialDeps = DepsSerializers.partialDepsById.deserialize((ByteBuffer) partialDeps);
+            }
+            catch (IOException e)
+            {
+                throw new IllegalStateException("Failed to materialise partially deserialised deps", e);
+            }
+        }
+        return (PartialDeps) partialDeps;
+    }
+
+    public void reset(JournalKey key)
+    {
+        reset(key.id);
+    }
+
+    public ByteBuffer asByteBuffer(Version userVersion) throws IOException
+    {
+        try (DataOutputBuffer out = new DataOutputBuffer())
+        {
+            serialize(out, userVersion);
+            return out.asNewBuffer();
+        }
+    }
+
+    public void serialize(DataOutputPlus out, Version userVersion) throws IOException
+    {
+        Invariants.require(mask == 0);
+        Invariants.require(flags != 0);
+
+        int flags = validateFlags(this.flags);
+        serialize(flags, out, userVersion);
+    }
+
+    private void serialize(int flags, DataOutputPlus out, Version userVersion) throws IOException
+    {
+        Invariants.require(flags != 0);
+        out.writeInt(flags);
+
+        int iterable = toIterableNonNullFields(flags);
+        for (CommandChange.Field field = nextSetField(iterable); field != null; iterable = unsetIterable(field, iterable), field = nextSetField(iterable))
+        {
+            switch (field)
+            {
+                default:
+                    throw new UnhandledEnum(field);
+                case CLEANUP:
+                    out.writeByte(cleanup.ordinal());
+                    break;
+                case EXECUTE_AT:
+                    Invariants.require(txnId != null, "%s", this);
+                    Invariants.require(executeAt != null, "%s", this);
+                    CommandSerializers.ExecuteAtSerializer.serialize(txnId, executeAt, out);
+                    break;
+                case EXECUTES_AT_LEAST:
+                    Invariants.require(executesAtLeast != null);
+                    CommandSerializers.ExecuteAtSerializer.serialize(executesAtLeast, out);
+                    break;
+                case MIN_UNIQUE_HLC:
+                    Invariants.require(minUniqueHlc != 0, "%s", this);
+                    out.writeUnsignedVInt(minUniqueHlc);
+                    break;
+                case SAVE_STATUS:
+                    Invariants.require(saveStatus != null, "%s", this);
+                    out.writeByte(saveStatus.ordinal());
+                    break;
+                case DURABILITY:
+                    Invariants.require(durability != null, "%s", this);
+                    out.writeByte(durability.encoded());
+                    break;
+                case ACCEPTED:
+                    Invariants.require(acceptedOrCommitted != null, "%s", this);
+                    CommandSerializers.ballot.serialize(acceptedOrCommitted, out);
+                    break;
+                case PROMISED:
+                    Invariants.require(promised != null, "%s", this);
+                    CommandSerializers.ballot.serialize(promised, out);
+                    break;
+                case PARTICIPANTS:
+                    Invariants.require(participants != null, "%s", this);
+                    CommandSerializers.participants.serialize(participants, out);
+                    break;
+                case PARTIAL_TXN:
+                    Invariants.require(partialTxn != null, "%s", this);
+                    CommandSerializers.partialTxn.serialize(partialTxn, out, userVersion);
+                    break;
+                case PARTIAL_DEPS:
+                    Invariants.require(partialDeps != null, "%s", this);
+                    if (partialDeps instanceof ByteBuffer) out.write(((ByteBuffer) partialDeps).duplicate());
+                    else DepsSerializers.partialDepsById.serialize((PartialDeps) partialDeps, out);
+                    break;
+                case WAITING_ON:
+                    Invariants.require(waitingOn != null, "%s", this);
+                    ((WaitingOnSerializer.WaitingOnBitSetsAndLength) waitingOn).reserialize(out);
+                    break;
+                case WRITES:
+                    Invariants.require(writes != null, "%s", this);
+                    CommandSerializers.writes.serialize(writes, out, userVersion);
+                    break;
+                case RESULT:
+                    Invariants.require(result != null, "%s", this);
+                    ResultSerializers.result.serialize(result, out);
+                    break;
+            }
+        }
+    }
+
+    public void deserializeNext(ByteBuffer buffer, Version userVersion)
+    {
+        try (DataInputBuffer in = new DataInputBuffer(buffer, false))
+        {
+            deserializeNext(in, userVersion);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public void deserializeNext(DataInputPlus in, Version userVersion) throws IOException
+    {
+        Invariants.require(txnId != null);
+        int readFlags = in.readInt();
+        Invariants.require(readFlags != 0);
+        hasUpdate = true;
+        count++;
+
+        // batch-apply any new nulls
+        setNulls(false, readFlags);
+        // iterator sets low 16 bits; low readFlag bits are nulls, so masking with ~readFlags restricts to non-null changed fields
+        int iterable = toIterableSetFields(readFlags) & ~readFlags;
+        for (CommandChange.Field field = nextSetField(iterable); field != null; field = nextSetField(iterable = unsetIterable(field, iterable)))
+        {
+            // Since we are iterating in reverse order, we skip the fields that were
+            // set by entries written later (i.e. already read ones).
+            if (isChanged(field, flags | mask) && field != CLEANUP)
+                skip(txnId, field, in, userVersion);
+            else
+                deserialize(field, in, userVersion);
+        }
+
+        // upper 16 bits are changed flags, lower are nulls; by masking upper by ~lower we restrict to only non-null changed fields
+        this.flags |= readFlags & (~readFlags << 16);
+    }
+
+    private void deserialize(CommandChange.Field field, DataInputPlus in, Version userVersion) throws IOException
+    {
+        switch (field)
+        {
+            case EXECUTE_AT:
+                executeAt = CommandSerializers.ExecuteAtSerializer.deserialize(txnId, in);
+                break;
+            case EXECUTES_AT_LEAST:
+                executesAtLeast = CommandSerializers.ExecuteAtSerializer.deserialize(in);
+                break;
+            case MIN_UNIQUE_HLC:
+                minUniqueHlc = in.readUnsignedVInt();
+                break;
+            case SAVE_STATUS:
+                saveStatus = SaveStatus.values()[in.readByte()];
+                break;
+            case DURABILITY:
+                durability = Status.Durability.forEncoded(in.readUnsignedByte());
+                break;
+            case ACCEPTED:
+                acceptedOrCommitted = CommandSerializers.ballot.deserialize(in);
+                break;
+            case PROMISED:
+                promised = CommandSerializers.ballot.deserialize(in);
+                break;
+            case PARTICIPANTS:
+                participants = CommandSerializers.participants.deserialize(in);
+                break;
+            case PARTIAL_TXN:
+                partialTxn = CommandSerializers.partialTxn.deserialize(in, userVersion);
+                break;
+            case PARTIAL_DEPS:
+                // TODO (expected): this optimisation will be easily disabled;
+                //  should either operate natively on ByteBuffer
+                //  or else use some explicit API for copying bytes while skipping
+                if (deserializeDeps || !(in instanceof DataInputBuffer))
+                {
+                    partialDeps = DepsSerializers.partialDepsById.deserialize(in);
+                }
+                else
+                {
+                    ByteBuffer buf = ((DataInputBuffer) in).buffer();
+                    int start = buf.position();
+                    DepsSerializers.partialDepsById.skip(in);
+                    int end = buf.position();
+                    partialDeps = buf.duplicate().position(start).limit(end);
+                }
+                break;
+            case WAITING_ON:
+                waitingOn = WaitingOnSerializer.deserializeBitSets(txnId, in);
+                break;
+            case WRITES:
+                writes = CommandSerializers.writes.deserialize(in, userVersion);
+                break;
+            case CLEANUP:
+                Cleanup newCleanup = Cleanup.forOrdinal(in.readByte());
+                if (cleanup == null || newCleanup.compareTo(cleanup) > 0)
+                    cleanup = newCleanup;
+                break;
+            case RESULT:
+                result = ResultSerializers.result.deserialize(in);
+                break;
+        }
+    }
+
+    private static void skip(TxnId txnId, CommandChange.Field field, DataInputPlus in, Version userVersion) throws IOException
+    {
+        switch (field)
+        {
+            default:
+                throw new UnhandledEnum(field);
+            case EXECUTE_AT:
+                CommandSerializers.ExecuteAtSerializer.skip(txnId, in);
+                break;
+            case EXECUTES_AT_LEAST:
+                CommandSerializers.ExecuteAtSerializer.skip(in);
+                break;
+            case MIN_UNIQUE_HLC:
+                in.readUnsignedVInt();
+                break;
+            case SAVE_STATUS:
+            case DURABILITY:
+            case CLEANUP:
+                in.readByte();
+                break;
+            case ACCEPTED:
+            case PROMISED:
+                CommandSerializers.ballot.skip(in);
+                break;
+            case PARTICIPANTS:
+                CommandSerializers.participants.skip(in);
+                break;
+            case PARTIAL_TXN:
+                CommandSerializers.partialTxn.skip(in, userVersion);
+                break;
+            case PARTIAL_DEPS:
+                DepsSerializers.partialDepsById.skip(in);
+                break;
+            case WAITING_ON:
+                WaitingOnSerializer.skip(txnId, in);
+                break;
+            case WRITES:
+                // TODO (expected): skip
+                CommandSerializers.writes.skip(in, userVersion);
+                break;
+            case RESULT:
+                // TODO (expected): skip
+                ResultSerializers.result.skip(in);
+                break;
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/DurableBeforePersister.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import accord.local.DurableBefore;
+import accord.primitives.TxnId;
+import accord.utils.PersistentField;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
+
+import org.apache.cassandra.service.accord.JournalKey;
+
+public class DurableBeforePersister implements PersistentField.Persister<DurableBefore, DurableBefore>
+{
+    private static final JournalKey JOURNAL_KEY = new JournalKey(TxnId.NONE, JournalKey.Type.DURABLE_BEFORE, 0);
+
+    final AccordJournal journal;
+
+    public DurableBeforePersister(AccordJournal journal)
+    {
+        this.journal = journal;
+    }
+
+    @Override
+    public AsyncResult<?> persist(DurableBefore addValue, DurableBefore newValue)
+    {
+        AsyncResult.Settable<Void> result = AsyncResults.settable();
+        journal.append(JOURNAL_KEY, addValue, () -> result.setSuccess(null));
+        return result;
+    }
+
+    @Override
+    public DurableBefore load()
+    {
+        MergeSerializers.DurableBeforeMerger accumulator = journal.readAll(JOURNAL_KEY);
+        return accumulator.get();
+    }
+};

src/java/org/apache/cassandra/service/accord/journal/IndexedRange.java

@@ -16,26 +16,26 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import org.apache.cassandra.cache.IMeasurableMemory;
 import org.apache.cassandra.utils.ByteArrayUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 
-public class IndexRange implements Comparable<IndexRange>, IMeasurableMemory
+class IndexedRange implements Comparable<IndexedRange>, IMeasurableMemory
 {
-    private static final long EMPTY_SIZE = ObjectSizes.measure(new IndexRange(null, null));
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new IndexedRange(null, null));
 
     public final byte[] start, end;
 
-    public IndexRange(byte[] start, byte[] end)
+    public IndexedRange(byte[] start, byte[] end)
     {
         this.start = start;
         this.end = end;
     }
 
     @Override
-    public int compareTo(IndexRange other)
+    public int compareTo(IndexedRange other)
     {
         int rc = ByteArrayUtil.compareUnsigned(start, 0, other.start, 0, start.length);
         if (rc == 0)

src/java/org/apache/cassandra/service/accord/journal/JournalRangeIndex.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import java.util.Map;
 import java.util.concurrent.CancellationException;
@@ -54,7 +54,12 @@ import accord.utils.UnhandledEnum;
 import accord.utils.btree.BTree;
 import accord.utils.btree.IntervalBTree;
 
+import org.apache.cassandra.service.accord.AccordCache;
+import org.apache.cassandra.service.accord.AccordCacheEntry;
+import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordCommandStore.Caches;
+import org.apache.cassandra.service.accord.RangeIndex;
+import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.TokenKey;
 
 import static accord.local.CommandSummaries.Relevance.IRRELEVANT;
@@ -194,7 +199,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
         }
 
         @Override
-        void finish(Map<Timestamp, Summary> into)
+        protected void finish(Map<Timestamp, Summary> into)
         {
         }
 
@@ -228,7 +233,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
         }
 
         @Override
-        void cleanupExclusive(Caches caches)
+        protected void cleanupExclusive(Caches caches)
         {
             if (commandWatcher != null)
             {
@@ -239,7 +244,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
         }
 
         @Override
-        AccordCommandStore commandStore()
+        protected AccordCommandStore commandStore()
         {
             return owner.commandStore;
         }
@@ -247,7 +252,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
 
     private final AccordCommandStore commandStore;
     // TODO (expected): do we need one of these per command store?
-    private final JournalRangeSearcher searcher;
+    private final RangeSearcher searcher;
     private final Object2ObjectHashMap<TxnId, RangeRoute> cachedRangeTxnsById = new Object2ObjectHashMap<>();
 
     public JournalRangeIndex(AccordCommandStore commandStore)
@@ -258,7 +263,7 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
         {
             caches.commands().register(this);
         }
-        this.searcher = JournalRangeSearcher.extractRangeSearcher(commandStore.journal);
+        this.searcher = RangeSearcher.extractRangeSearcher(commandStore.journal);
     }
 
     @Override
@@ -366,16 +371,6 @@ public class JournalRangeIndex extends SemiSyncIntervalTree<Object[]> implements
         return SummaryLoader.loader(redundantBefore, maxDecidedRX, primaryTxnId, primaryExecuteAt, loadKeysFor, keysOrRanges, this::newLoader);
     }
 
-    @Override
-    public void update(Command prev, Command updated, boolean force)
-    {
-    }
-
-    @Override
-    public void postReplay()
-    {
-    }
-
     private Loader newLoader(RedundantBefore redundantBefore, MaxDecidedRX maxDecidedRX, @Nullable TxnId primaryTxnId, Unseekables<?> searchKeysOrRanges, Kinds testKind, TxnId minTxnId, Timestamp maxTxnId, LoadKeysFor loadKeysFor)
     {
         return new Loader(this, redundantBefore, maxDecidedRX, primaryTxnId, searchKeysOrRanges, testKind, minTxnId, maxTxnId, loadKeysFor);

src/java/org/apache/cassandra/service/accord/journal/MergeSerializer.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.serializers.Version;
+
+public interface MergeSerializer<V, DeserializeInto extends Merger, M extends DeserializeInto>
+{
+    M mergerFor();
+
+    void deserialize(JournalKey key, DeserializeInto into, DataInputPlus in, Version userVersion) throws IOException;
+
+    default M deserialize(JournalKey key, DataInputPlus in, Version userVersion) throws IOException
+    {
+        M builder = mergerFor();
+        deserialize(key, builder, in, userVersion);
+        return builder;
+    }
+
+    void serialize(JournalKey key, V from, DataOutputPlus out, Version userVersion) throws IOException;
+
+    void reserialize(JournalKey key, M from, DataOutputPlus out, Version userVersion) throws IOException;
+
+
+}

src/java/org/apache/cassandra/service/accord/journal/MergeSerializers.java

@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import com.google.common.collect.ImmutableSortedMap;
+
+import accord.local.DurableBefore;
+import accord.local.RedundantBefore;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.journal.Merger.KeepFirst;
+import org.apache.cassandra.service.accord.journal.Merger.SimpleMerger;
+import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers;
+import org.apache.cassandra.service.accord.serializers.Version;
+
+import static accord.local.CommandStores.RangesForEpoch;
+
+public class MergeSerializers
+{
+    public static class CommandChangeSerializer implements MergeSerializer<CommandChangeWriter, CommandChanges, CommandChanges>
+    {
+        @Override
+        public CommandChanges mergerFor()
+        {
+            return new CommandChanges();
+        }
+
+        @Override
+        public void serialize(JournalKey key, CommandChangeWriter writer, DataOutputPlus out, Version userVersion)
+        {
+            try
+            {
+                writer.write(out, userVersion);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void reserialize(JournalKey key, CommandChanges from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            from.serialize(out,
+                           // In CompactionIterator, we are dealing with relatively recent records, so we do not pass redundant before here.
+                           // However, we do on load and during Journal SSTable compaction.
+                           userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey journalKey, CommandChanges into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            into.deserializeNext(in, userVersion);
+        }
+    }
+
+    public static class RedundantBeforeSerializer
+    implements MergeSerializer<RedundantBefore,
+                              SimpleMerger<?, ? super RedundantBefore>,
+                              SimpleMerger<RedundantBefore, RedundantBefore>>
+    {
+        @Override
+        public KeepFirst<RedundantBefore> mergerFor()
+        {
+            return new KeepFirst<>(RedundantBefore.EMPTY);
+        }
+
+        @Override
+        public void serialize(JournalKey key, RedundantBefore entry, DataOutputPlus out, Version userVersion)
+        {
+            try
+            {
+                if (entry == RedundantBefore.EMPTY)
+                {
+                    // I am fairly sure this branch was to paper over a bug in the RedundantBefore serializer; it should now be defunct
+                    out.writeInt(0);
+                    return;
+                }
+                out.writeInt(1);
+                CommandStoreSerializers.redundantBefore.serialize(entry, out);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void reserialize(JournalKey key, SimpleMerger<RedundantBefore, RedundantBefore> from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            serialize(key, from.get(), out, userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey journalKey, SimpleMerger<?, ? super RedundantBefore> into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            if (in.readInt() == 0)
+            {
+                into.update(RedundantBefore.EMPTY);
+                return;
+            }
+            into.update(CommandStoreSerializers.redundantBefore.deserialize(in));
+        }
+    }
+
+    public static class DurableBeforeMerger extends SimpleMerger<DurableBefore, DurableBefore>
+    {
+        public DurableBeforeMerger()
+        {
+            super(DurableBefore.EMPTY);
+        }
+
+        @Override
+        public void reset(JournalKey key)
+        {
+            accumulated = DurableBefore.EMPTY;
+        }
+
+        @Override
+        protected DurableBefore merge(DurableBefore oldValue, DurableBefore newValue)
+        {
+            return DurableBefore.merge(oldValue, newValue);
+        }
+    }
+
+    public static class DurableBeforeSerializer
+    implements MergeSerializer<DurableBefore,
+                              SimpleMerger<?, ? super DurableBefore>,
+                              DurableBeforeMerger>
+    {
+        public DurableBeforeMerger mergerFor()
+        {
+            return new DurableBeforeMerger();
+        }
+
+        @Override
+        public void serialize(JournalKey key, DurableBefore entry, DataOutputPlus out, Version userVersion)
+        {
+            try
+            {
+                CommandStoreSerializers.durableBefore.serialize(entry, out);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void reserialize(JournalKey key, DurableBeforeMerger from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            serialize(key, from.get(), out, userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey journalKey, SimpleMerger<?, ? super DurableBefore> into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            into.update(CommandStoreSerializers.durableBefore.deserialize(in));
+        }
+    }
+
+    public static class BootstrapBeganAtSerializer
+    implements MergeSerializer<NavigableMap<TxnId, Ranges>,
+                              SimpleMerger<?, ? super NavigableMap<TxnId, Ranges>>,
+                              SimpleMerger<NavigableMap<TxnId, Ranges>, NavigableMap<TxnId, Ranges>>>
+    {
+        @Override
+        public KeepFirst<NavigableMap<TxnId, Ranges>> mergerFor()
+        {
+            return new KeepFirst<>(ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY));
+        }
+
+        @Override
+        public void serialize(JournalKey key, NavigableMap<TxnId, Ranges> entry, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            CommandStoreSerializers.bootstrapBeganAt.serialize(entry, out);
+        }
+
+        @Override
+        public void reserialize(JournalKey key, SimpleMerger<NavigableMap<TxnId, Ranges>, NavigableMap<TxnId, Ranges>> image, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            serialize(key, image.get(), out, userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey key, SimpleMerger<?, ? super NavigableMap<TxnId, Ranges>> into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            into.update(CommandStoreSerializers.bootstrapBeganAt.deserialize(in));
+        }
+    }
+
+    public static class SafeToReadSerializer
+    implements MergeSerializer<NavigableMap<Timestamp, Ranges>,
+                              SimpleMerger<?, ? super NavigableMap<Timestamp, Ranges>>,
+                              SimpleMerger<NavigableMap<Timestamp, Ranges>, NavigableMap<Timestamp, Ranges>>>
+    {
+        @Override
+        public KeepFirst<NavigableMap<Timestamp, Ranges>> mergerFor()
+        {
+            return new KeepFirst<>(ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY));
+        }
+
+        @Override
+        public void serialize(JournalKey key, NavigableMap<Timestamp, Ranges> from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            CommandStoreSerializers.safeToRead.serialize(from, out);
+        }
+
+        @Override
+        public void reserialize(JournalKey key, SimpleMerger<NavigableMap<Timestamp, Ranges>, NavigableMap<Timestamp, Ranges>> from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            serialize(key, from.get(), out, userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey key, SimpleMerger<?, ? super NavigableMap<Timestamp, Ranges>> into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            into.update(CommandStoreSerializers.safeToRead.deserialize(in));
+        }
+    }
+
+    public static class RangesForEpochSerializer
+    implements MergeSerializer<RangesForEpoch,
+                              SimpleMerger<?, ? super RangesForEpoch>,
+                              SimpleMerger<RangesForEpoch, RangesForEpoch>>
+    {
+        public static final RangesForEpochSerializer instance = new RangesForEpochSerializer();
+        public KeepFirst<RangesForEpoch> mergerFor()
+        {
+            return new KeepFirst<>(null);
+        }
+
+        @Override
+        public void serialize(JournalKey key, RangesForEpoch from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            CommandStoreSerializers.rangesForEpoch.serialize(from, out);
+        }
+
+        @Override
+        public void reserialize(JournalKey key, SimpleMerger<RangesForEpoch, RangesForEpoch> from, DataOutputPlus out, Version userVersion) throws IOException
+        {
+            serialize(key, from.get(), out, userVersion);
+        }
+
+        @Override
+        public void deserialize(JournalKey key, SimpleMerger<?, ? super RangesForEpoch> into, DataInputPlus in, Version userVersion) throws IOException
+        {
+            into.update(CommandStoreSerializers.rangesForEpoch.deserialize(in));
+        }
+    }
+
+    public static class TopologySerializer implements MergeSerializer<TopologyRecord, TopologyMerger, TopologyMerger>
+    {
+        public static final TopologySerializer INSTANCE = new TopologySerializer();
+
+        public TopologySerializer() {}
+
+        @Override
+        public TopologyMerger mergerFor()
+        {
+            return new TopologyMerger();
+        }
+
+        @Override
+        public void serialize(JournalKey key, TopologyRecord from, DataOutputPlus out, Version version) throws IOException
+        {
+            TopologyRecord.Serializer.instance.serialize(from, out);
+        }
+
+        @Override
+        public void reserialize(JournalKey key, TopologyMerger from, DataOutputPlus out, Version version) throws IOException
+        {
+            serialize(key, from.write, out, version);
+        }
+
+        @Override
+        public void deserialize(JournalKey key, TopologyMerger into, DataInputPlus in, Version version) throws IOException
+        {
+            into.read(TopologyRecord.Serializer.instance.deserialize(in));
+        }
+    }
+
+    public static class TopologyMerger implements Merger
+    {
+        TopologyRecord.TopologyImage read, write;
+
+        public TopologyMerger()
+        {
+        }
+
+        @Override
+        public void reset(JournalKey key)
+        {
+            read = write = null;
+        }
+
+        public TopologyRecord.TopologyImage read()
+        {
+            return read;
+        }
+
+        public void read(TopologyRecord update)
+        {
+            if (Objects.requireNonNull(update.kind()) == TopologyRecord.Kind.New)
+                read = new TopologyRecord.TopologyImage(update.epoch(), TopologyRecord.Kind.Image, update.getUpdate());
+            else
+                read = (TopologyRecord.TopologyImage) update;
+            write = read;
+        }
+
+        public void write(TopologyRecord.TopologyImage image)
+        {
+            Invariants.require(write == read);
+            this.write = image;
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/Merger.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.service.accord.JournalKey;
+
+public interface Merger
+{
+    abstract class SimpleMerger<A, V> implements Merger
+    {
+        protected A accumulated;
+
+        public SimpleMerger(A initial) {this.accumulated = initial; }
+        public void update(V newValue) { accumulated = merge(accumulated, newValue); }
+        public A get() { return accumulated; }
+
+        protected abstract A merge(A oldValue, V newValue);
+    }
+
+    class KeepFirst<V> extends SimpleMerger<V, V>
+    {
+        final V ifNone;
+        boolean hasRead;
+        public KeepFirst(V ifNone)
+        {
+            super(ifNone);
+            this.ifNone = ifNone;
+        }
+
+        @Override
+        public void reset(JournalKey key)
+        {
+            hasRead = false;
+            accumulated = ifNone;
+        }
+
+        @Override
+        protected V merge(V oldValue, V newValue)
+        {
+            if (hasRead)
+                return oldValue;
+            hasRead = true;
+            return newValue;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "KeepFirst{" + accumulated + '}';
+        }
+    }
+
+    class KeepList<V> extends SimpleMerger<List<V>, V>
+    {
+        public KeepList(List<V> initial)
+        {
+            super(initial);
+        }
+
+        public KeepList()
+        {
+            super(new ArrayList<>());
+        }
+
+        @Override
+        protected List<V> merge(List<V> oldValue, V newValue)
+        {
+            oldValue.add(newValue);
+            return oldValue;
+        }
+
+        @Override
+        public void reset(JournalKey key)
+        {
+            accumulated.clear();
+        }
+    }
+
+    void reset(JournalKey key);
+}

src/java/org/apache/cassandra/service/accord/journal/RangeSearchManager.java

@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.accord.journal;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.local.MaxDecidedRX;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import accord.utils.UncheckedInterruptedException;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.accord.OrderedRouteSerializer;
+import org.apache.cassandra.index.accord.RouteJournalIndex;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.journal.StaticSegment;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.service.RetryStrategy;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.TokenKey;
+import org.apache.cassandra.service.accord.serializers.CommandSerializers;
+import org.apache.cassandra.service.accord.serializers.Version;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.MergeIterator;
+
+import static org.apache.cassandra.service.accord.AccordKeyspace.JournalColumns.getJournalKey;
+
+public class RangeSearchManager implements RangeSearcher.Supplier
+{
+    private static final Logger logger = LoggerFactory.getLogger(RangeSearchManager.class);
+
+    final ColumnFamilyStore cfs;
+
+    /**
+     * Access to this field should only ever be handled by {@link #safeNotify(Consumer)}.  There is an assumption that
+     * an error in the index should not cause the journal to crash, so {@link #safeNotify(Consumer)} exists to make sure
+     * this property holds true.
+     */
+    @Nullable
+    private final SegmentRangeSearcher<Object> segmentSearcher;
+
+    private RangeSearchManager(ColumnFamilyStore cfs)
+    {
+        this.cfs = cfs;
+        this.segmentSearcher = new SegmentRangeSearcher<>();
+    }
+
+    static @Nullable RangeSearchManager ifEnabled(ColumnFamilyStore cfs)
+    {
+        return cfs.indexManager.getIndexByName(AccordKeyspace.JOURNAL_INDEX_NAME) != null ?
+               new RangeSearchManager(cfs) : null;
+    }
+
+    org.apache.cassandra.journal.SegmentCompactor<JournalKey, Object> compactor(ColumnFamilyStore cfs, Version userVersion)
+    {
+        return new SegmentCompactor<>(userVersion, cfs) {
+            @Nullable
+            @Override
+            public Collection<StaticSegment<JournalKey, Object>> compact(Collection<StaticSegment<JournalKey, Object>> staticSegments)
+            {
+                Collection<StaticSegment<JournalKey, Object>> result = super.compact(staticSegments);
+                safeNotify(index -> index.remove(staticSegments));
+                return result;
+            }
+        };
+    }
+
+    Runnable maybeIndex(JournalKey key, RecordPointer pointer, CommandChangeWriter change)
+    {
+        if (shouldIndex(key)
+            && change.hasParticipants()
+            && change.after.route() != null)
+        {
+            return () -> safeNotify(index -> index.update(pointer.segment, key.commandStoreId, key.id, change.after.route()));
+        }
+        return null;
+    }
+
+    boolean shouldIndex(JournalKey key)
+    {
+        return RouteJournalIndex.allowed(key);
+    }
+
+    void safeNotify(Consumer<SegmentRangeSearcher<Object>> fn)
+    {
+        if (segmentSearcher == null)
+            return;
+        try
+        {
+            fn.accept(segmentSearcher);
+        }
+        catch (Throwable t)
+        {
+            JVMStabilityInspector.inspectThrowable(t);
+            logger.warn("Failure updating index", t);
+        }
+    }
+
+    @Override
+    public RangeSearcher rangeSearcher()
+    {
+        return new JournalTableRangeSearcher();
+    }
+
+    public void start()
+    {
+        Index tableIndex = cfs.indexManager.getIndexByName(AccordKeyspace.JOURNAL_INDEX_NAME);
+        RetryStrategy retry = DatabaseDescriptor.getAccord().retry_journal_index_ready.retry();
+        for (int i = 0; !cfs.indexManager.isIndexQueryable(tableIndex); i++)
+        {
+            logger.debug("Journal index {} is not ready wait... waiting", AccordKeyspace.JOURNAL_INDEX_NAME);
+            maybeWait(retry, i);
+        }
+    }
+
+    /**
+     * This method is here to make it easier for org.apache.cassandra.distributed.test.accord.journal.JournalAccessRouteIndexOnStartupRaceTest
+     * to check when we need to do waiting
+     */
+    @VisibleForTesting
+    private static void maybeWait(RetryStrategy retry, int i)
+    {
+        long waitTime = retry.computeWait(i, TimeUnit.MICROSECONDS);
+        if (waitTime == -1)
+            throw new IllegalStateException("Gave up waiting on journal index to be ready");
+        try
+        {
+            TimeUnit.MICROSECONDS.sleep(waitTime);
+        }
+        catch (InterruptedException e)
+        {
+            throw new UncheckedInterruptedException(e);
+        }
+    }
+
+    /**
+     * When using {@link PartitionRangeReadCommand} we need to work with {@link RowFilter} which works with columns.
+     * But the index doesn't care about table based queries and needs to be queried using the fields in the index, to
+     * support that this enum exists.  This enum represents the fields present in the index and can be used to apply
+     * filters to the index.
+     */
+    public enum SyntheticColumn
+    {
+        participants("participants", BytesType.instance),
+        store_id("store_id", Int32Type.instance),
+        txn_id("txn_id", BytesType.instance);
+
+        public final ColumnMetadata metadata;
+
+        SyntheticColumn(String name, AbstractType<?> type)
+        {
+            this.metadata = new ColumnMetadata("journal", "routes", new ColumnIdentifier(name, false), type, ColumnMetadata.NO_UNIQUE_ID, ColumnMetadata.NO_POSITION, ColumnMetadata.Kind.REGULAR, null);
+        }
+    }
+
+    private class JournalTableRangeSearcher implements RangeSearcher
+    {
+        private final Index tableIndex;
+
+        private JournalTableRangeSearcher()
+        {
+            this.tableIndex = cfs.indexManager.getIndexByName("record");
+            if (!cfs.indexManager.isIndexQueryable(tableIndex))
+                throw new AssertionError("Journal record index is not queryable");
+        }
+
+        @Override
+        public Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
+        {
+            CloseableIterator<TxnId> inMemory = segmentSearcher.search(commandStoreId, range, minTxnId, maxTxnId, decidedRX).results();
+            CloseableIterator<TxnId> table = tableSearch(commandStoreId, range.start(), range.end(), minTxnId, maxTxnId, decidedRX);
+            return new DefaultResult(minTxnId, maxTxnId, decidedRX, MergeIterator.get(Arrays.asList(inMemory, table)));
+        }
+
+        @Override
+        public Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
+        {
+            CloseableIterator<TxnId> inMemory = segmentSearcher.search(commandStoreId, key, minTxnId, maxTxnId, decidedRX).results();
+            CloseableIterator<TxnId> table = tableSearch(commandStoreId, key, minTxnId, maxTxnId);
+            return new DefaultResult(minTxnId, maxTxnId, decidedRX, MergeIterator.get(Arrays.asList(inMemory, table)));
+        }
+
+        private CloseableIterator<TxnId> tableSearch(int store, TokenKey start, TokenKey end, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
+        {
+            RowFilter rowFilter = RowFilter.create(false);
+            rowFilter.add(RangeSearchManager.SyntheticColumn.participants.metadata, Operator.GT, OrderedRouteSerializer.serialize(start));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(end));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.txn_id.metadata, Operator.GTE, CommandSerializers.txnId.serialize(minTxnId));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.txn_id.metadata, Operator.LTE, CommandSerializers.timestamp.serialize(maxTxnId));
+            return process(store, rowFilter);
+        }
+
+        private CloseableIterator<TxnId> tableSearch(int store, TokenKey key, TxnId minTxnId, Timestamp maxTxnId)
+        {
+            RowFilter rowFilter = RowFilter.create(false);
+            rowFilter.add(RangeSearchManager.SyntheticColumn.participants.metadata, Operator.GTE, OrderedRouteSerializer.serialize(key));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(key));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.txn_id.metadata, Operator.GTE, CommandSerializers.txnId.serialize(minTxnId));
+            rowFilter.add(RangeSearchManager.SyntheticColumn.txn_id.metadata, Operator.LTE, CommandSerializers.timestamp.serialize(maxTxnId));
+            return process(store, rowFilter);
+        }
+
+        private CloseableIterator<TxnId> process(int storeId, RowFilter rowFilter)
+        {
+            PartitionRangeReadCommand cmd = PartitionRangeReadCommand.create(cfs.metadata(),
+                                                                             FBUtilities.nowInSeconds(),
+                                                                             ColumnFilter.selectionBuilder()
+                                                                                         .add(RangeSearchManager.SyntheticColumn.store_id.metadata)
+                                                                                         .add(RangeSearchManager.SyntheticColumn.txn_id.metadata)
+                                                                                         .build(),
+                                                                             rowFilter,
+                                                                             DataLimits.NONE,
+                                                                             DataRange.allData(cfs.getPartitioner()));
+            Index.Searcher s = tableIndex.searcherFor(cmd);
+            try (ReadExecutionController controller = cmd.executionController())
+            {
+                UnfilteredPartitionIterator partitionIterator = s.search(controller);
+                return new CloseableIterator<>()
+                {
+
+                    @Override
+                    public void close()
+                    {
+                        partitionIterator.close();
+                    }
+
+                    @Override
+                    public boolean hasNext()
+                    {
+                        return partitionIterator.hasNext();
+                    }
+
+                    @Override
+                    public TxnId next()
+                    {
+                        UnfilteredRowIterator next = partitionIterator.next();
+                        JournalKey partitionKeyComponents = getJournalKey(next.partitionKey());
+                        Invariants.require(partitionKeyComponents.commandStoreId == storeId,
+                                              () -> String.format("table index returned a command store other than the expected one; expected %d != %d", storeId, partitionKeyComponents.commandStoreId));
+                        return partitionKeyComponents.id;
+                    }
+                };
+            }
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/RangeSearcher.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import java.util.function.Consumer;
 
@@ -27,24 +27,25 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 
 import org.apache.cassandra.index.accord.RouteIndexFormat;
+import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.TokenKey;
 import org.apache.cassandra.utils.CloseableIterator;
 
-public interface JournalRangeSearcher
+public interface RangeSearcher
 {
     Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX);
     Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX);
 
-    static JournalRangeSearcher extractRangeSearcher(Object o)
+    static RangeSearcher extractRangeSearcher(Object o)
     {
-        if (o instanceof JournalRangeSearcher.Supplier)
-            return ((JournalRangeSearcher.Supplier) o).rangeSearcher();
+        if (o instanceof RangeSearcher.Supplier)
+            return ((RangeSearcher.Supplier) o).rangeSearcher();
         return NoopJournalRangeSearcher.instance;
     }
 
     interface Supplier
     {
-        JournalRangeSearcher rangeSearcher();
+        RangeSearcher rangeSearcher();
     }
 
     interface Result
@@ -117,7 +118,7 @@ public interface JournalRangeSearcher
         }
     }
 
-    enum NoopJournalRangeSearcher implements JournalRangeSearcher
+    enum NoopJournalRangeSearcher implements RangeSearcher
     {
         instance;
 

src/java/org/apache/cassandra/service/accord/journal/Replay.java

@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntArrayList;
+import org.agrona.collections.Long2LongHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.impl.AbstractReplayer;
+import accord.local.CommandStore;
+import accord.primitives.Route;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import accord.utils.UnhandledEnum;
+
+import org.apache.cassandra.config.AccordSpec.JournalSpec.ReplayMode;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.journal.Journal;
+import org.apache.cassandra.service.accord.AccordCommandStore;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.utils.Closeable;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.concurrent.Semaphore;
+
+import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE;
+import static accord.local.RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE;
+import static org.apache.cassandra.service.accord.JournalKey.Type.COMMAND_DIFF;
+import static org.apache.cassandra.utils.FBUtilities.getAvailableProcessors;
+
+public class Replay
+{
+    private static final Logger logger = LoggerFactory.getLogger(Replay.class);
+
+    static boolean replay(AccordJournal journal, ReplayMode replayMode, CommandStore[] commandStores, Object param)
+    {
+        AbstractReplayer.Mode accordReplayerMode;
+        switch (replayMode)
+        {
+            default: throw new UnhandledEnum(replayMode);
+            case NON_DURABLE:
+                accordReplayerMode = AbstractReplayer.Mode.NON_DURABLE;
+                throw new UnsupportedOperationException("Not yet safe to use NON_DURABLE ReplayMode");
+            case PART_NON_DURABLE:
+                accordReplayerMode = AbstractReplayer.Mode.PART_NON_DURABLE;
+                break;
+            case ALL:
+            case RESET:
+                accordReplayerMode = AbstractReplayer.Mode.ALL;
+        }
+
+        Invariants.require(param == null || param.getClass() == Long2LongHashMap.class, "Param should be null or a map of commandStoreId->minSegmentId");
+        final Long2LongHashMap minSegments = param == null || journal.rangeSearch != null ? new Long2LongHashMap(0L) : (Long2LongHashMap) param;
+        if (journal.rangeSearch != null)
+            logger.warn("journal_sai index enabled, which means safe replay markers do not reduce replay work");
+
+        // TODO (expected): make the parallelisms configurable
+        // Replay is performed in parallel, where at most X commands can be in flight, across at most Y commands stores.
+        // That is, you can limit replay parallelism to 1 command store at a time, but load multiple commands within that data store,
+        // _or_ have multiple commands being loaded accross multiple data stores.
+        final Semaphore commandParallelism = Semaphore.newSemaphore(getAvailableProcessors());
+        final int commandStoreParallelism = Math.max(Math.max(1, Math.min(getAvailableProcessors(), 4)), getAvailableProcessors() / 4);
+        final AtomicBoolean abort = new AtomicBoolean();
+        final IntArrayList activeCommandStoreIds = new IntArrayList();
+        final ReplayQueue pendingCommandStores = new ReplayQueue(commandStores);
+
+        class ReplayStream implements Closeable
+        {
+            final CommandStore commandStore;
+            final AbstractReplayer replayer;
+            final CloseableIterator<Journal.KeyRefs<JournalKey>> iter;
+            JournalKey prev;
+
+            public ReplayStream(CommandStore commandStore, long minSegment)
+            {
+                this.commandStore = commandStore;
+                this.replayer = (AbstractReplayer) commandStore.replayer(accordReplayerMode);
+                // Keys in the index are sorted by command store id, so index iteration will be sequential
+                this.iter = journal.keyIterator(new JournalKey(replayer.minReplay.withoutNonIdentityFlags(), COMMAND_DIFF, commandStore.id()), new JournalKey(TxnId.MAX.withoutNonIdentityFlags(), COMMAND_DIFF, commandStore.id()), false, minSegment);
+                logger.info("Beginning replay of {} with min={}, {}", commandStore, replayer.minReplay,
+                            replayer.redundantBefore.map(b -> b == null ? null : b.maxBoundBoth(LOCALLY_DURABLE_TO_DATA_STORE, LOCALLY_DURABLE_TO_COMMAND_STORE), TxnId[]::new));
+            }
+
+            boolean replay()
+            {
+                JournalKey key;
+                long[] segments;
+                while (true)
+                {
+                    if (!iter.hasNext())
+                    {
+                        logger.info("Completed replay of {}", commandStore);
+                        return false;
+                    }
+
+                    Journal.KeyRefs<JournalKey> ref = iter.next();
+                    if (ref.key().type != COMMAND_DIFF)
+                        continue;
+
+                    key = ref.key();
+                    segments = journal.rangeSearch != null && journal.rangeSearch.shouldIndex(key) ? ref.copyOfSegments() : null;
+                    break;
+                }
+
+                TxnId txnId = key.id;
+                Invariants.require(prev == null ||
+                                   key.commandStoreId != prev.commandStoreId ||
+                                   key.id.compareTo(prev.id) != 0,
+                                   "duplicate key detected %s == %s", key, prev);
+                prev = key;
+                commandParallelism.acquireThrowUncheckedOnInterrupt(1);
+                replayer.replay(txnId)
+                        .map(route -> {
+                            if (segments != null && route != null)
+                            {
+                                for (long segment : segments)
+                                    journal.rangeSearch.safeNotify(index -> index.update(segment, key.commandStoreId, txnId, (Route<?>) route));
+                            }
+                            return null;
+                        }).begin((success, fail) -> {
+                            commandParallelism.release(1);
+                            if (fail != null && !journal.segments.handleError("Could not replay command " + txnId, fail))
+                                abort.set(true);
+                        });
+
+                return true;
+            }
+
+            @Override
+            public void close()
+            {
+                iter.close();
+            }
+        }
+
+        // Replay streams by command store id, can hold at most commandStoreParallelism items
+        final Int2ObjectHashMap<ReplayStream> replayStreams = new Int2ObjectHashMap<>();
+        try
+        {
+            // index of the store we're currently pulling from in the activeCommandStoreIds collection
+            int cur = 0;
+            while (!abort.get())
+            {
+                if (cur == activeCommandStoreIds.size())
+                {
+                    if (activeCommandStoreIds.size() < commandStoreParallelism && !pendingCommandStores.isEmpty())
+                    {
+                        CommandStore next = pendingCommandStores.next();
+                        int id = next.id();
+                        activeCommandStoreIds.add(id);
+                        replayStreams.put(id, new ReplayStream(next, minSegments.getOrDefault(id, 0)));
+                    }
+                    else if (activeCommandStoreIds.isEmpty()) break;
+                    else cur = 0;
+                }
+
+                int id = activeCommandStoreIds.get(cur);
+                ReplayStream replayStream = replayStreams.get(id);
+                while (!replayStream.replay())
+                {
+                    // Replay complete for this command store; close and replace
+                    replayStreams.remove(id).close();
+                    if (pendingCommandStores.isEmpty())
+                    {
+                        // no more pending to submit; remove and continue with the next remaining (if any)
+                        activeCommandStoreIds.removeAt(cur);
+                        if (cur == activeCommandStoreIds.size())
+                            --cur;
+                        if (cur < 0)
+                            break;
+                        id = activeCommandStoreIds.get(cur);
+                    }
+                    else
+                    {
+                        // replace it with a pending command store, and continue processing
+                        CommandStore next = pendingCommandStores.next(streamId(replayStream.commandStore));
+                        id = next.id();
+                        activeCommandStoreIds.set(cur, id);
+                        replayStreams.put(id, new ReplayStream(next, minSegments.getOrDefault(id, 0)));
+                    }
+
+                    replayStream = replayStreams.get(id);
+                }
+
+                ++cur;
+            }
+            return true;
+        }
+        catch (Throwable t)
+        {
+            try { FileUtils.close(replayStreams.values()); }
+            catch (Throwable t2) { t.addSuppressed(t2); }
+            throw t;
+        }
+    }
+
+    static class ReplayQueue
+    {
+        final Int2ObjectHashMap<Queue<CommandStore>> byExecutor = new Int2ObjectHashMap<>();
+        final Deque<Integer> nextId = new ArrayDeque<>();
+
+        ReplayQueue(CommandStore[] commandStores)
+        {
+            for (CommandStore commandStore : commandStores)
+            {
+                byExecutor.computeIfAbsent(streamId(commandStore), ignore -> new ArrayDeque<>())
+                          .add(commandStore);
+            }
+            nextId.addAll(byExecutor.keySet());
+        }
+
+        boolean isEmpty()
+        {
+            return byExecutor.isEmpty();
+        }
+
+        CommandStore next()
+        {
+            while (true)
+            {
+                if (byExecutor.isEmpty())
+                    return null;
+
+                Integer id = nextId.poll();
+                if (id == null)
+                {
+                    nextId.addAll(byExecutor.keySet());
+                    id = nextId.poll();
+                }
+
+                Queue<CommandStore> queue = byExecutor.get(id);
+                if (queue != null)
+                {
+                    CommandStore next = queue.poll();
+                    if (queue.isEmpty())
+                        byExecutor.remove(id);
+                    if (next != null)
+                        return next;
+                }
+            }
+        }
+
+        CommandStore next(int streamId)
+        {
+            Queue<CommandStore> queue = byExecutor.get(streamId);
+            if (queue == null)
+                return next();
+
+            CommandStore next = queue.poll();
+            if (queue.isEmpty())
+                byExecutor.remove(streamId);
+
+            return next;
+        }
+    }
+
+    private static int streamId(CommandStore commandStore)
+    {
+        return commandStore instanceof AccordCommandStore ? ((AccordCommandStore) commandStore).executor().executorId() : 1;
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/ReplayMarkers.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileInputStreamPlus;
+import org.apache.cassandra.io.util.FileOutputStreamPlus;
+import org.apache.cassandra.utils.NativeLibrary;
+
+import static org.apache.cassandra.config.DatabaseDescriptor.getAccordJournalDirectory;
+
+public class ReplayMarkers
+{
+    public static File startMarker()
+    {
+        return new File(getAccordJournalDirectory(), "started");
+    }
+
+    public static File safeStopMarker()
+    {
+        return new File(getAccordJournalDirectory(), "stopped");
+    }
+
+    // TODO (required): add checksummed version and default to this (but support unchecksummed for manual editing)
+    static void writeMarker(File file, long timestamp)
+    {
+        try (FileOutputStreamPlus out = new FileOutputStreamPlus(file))
+        {
+            out.writeBytes(Long.toString(timestamp));
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+        trySyncJournalDirectory();
+    }
+
+    public static long readStartMarker()
+    {
+        return readMarker(startMarker());
+    }
+
+    public static long readStopMarker()
+    {
+        return readMarker(safeStopMarker());
+    }
+
+    public static long readMarker(File file)
+    {
+        if (!file.exists())
+            return -1L;
+
+        try (FileInputStreamPlus in = new FileInputStreamPlus(file))
+        {
+            StringBuilder sb = new StringBuilder(8);
+            for (int b = in.read(); b >= 0 ; b = in.read())
+                sb.append((char)b);
+            return Long.parseLong(sb.toString());
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static void trySyncJournalDirectory()
+    {
+        trySyncDirectory(getAccordJournalDirectory());
+    }
+
+    private static void trySyncDirectory(String path)
+    {
+        int fd = NativeLibrary.tryOpenDirectory(path);
+        NativeLibrary.trySync(fd);
+    }
+
+    public static File saveDirectory()
+    {
+        return new File(getAccordJournalDirectory(), "save_state");
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/SegmentCompactor.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
@@ -27,11 +27,11 @@ import org.apache.cassandra.service.accord.serializers.Version;
 /**
  * Segment compactor: takes static segments and compacts them into a single SSTable.
  */
-public class AccordSegmentCompactor<V> extends AbstractAccordSegmentCompactor<V>
+public class SegmentCompactor<V> extends AbstractSegmentCompactor<V>
 {
     private SSTableTxnWriter writer;
 
-    public AccordSegmentCompactor(Version userVersion, ColumnFamilyStore cfs)
+    public SegmentCompactor(Version userVersion, ColumnFamilyStore cfs)
     {
         super(userVersion, cfs);
     }

src/java/org/apache/cassandra/service/accord/journal/SegmentRangeSearcher.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.journal;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -49,6 +49,8 @@ import org.apache.cassandra.index.accord.RouteIndexFormat;
 import org.apache.cassandra.index.accord.RouteJournalIndex;
 import org.apache.cassandra.journal.StaticSegment;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.TokenKey;
 import org.apache.cassandra.utils.ByteArrayUtil;
 import org.apache.cassandra.utils.CloseableIterator;
@@ -58,7 +60,7 @@ import org.apache.cassandra.utils.RangeTree;
 
 import static accord.primitives.Routable.Domain.Range;
 
-public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
+public class SegmentRangeSearcher<V> implements RangeSearcher
 {
     private final Long2ObjectHashMap<SegmentIndex> segmentIndexes = new Long2ObjectHashMap<>();
 
@@ -85,7 +87,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
     }
 
     @Override
-    public JournalRangeSearcher.Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX)
+    public RangeSearcher.Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX)
     {
         NavigableSet<TxnId> result = search(commandStoreId, range.table(),
                                             OrderedRouteSerializer.serializeTokenOnly(range.start()),
@@ -103,7 +105,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
     }
 
     @Override
-    public JournalRangeSearcher.Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
+    public RangeSearcher.Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX)
     {
         NavigableSet<TxnId> result = search(commandStoreId, key.table(), OrderedRouteSerializer.serializeTokenOnly(key), minTxnId, maxTxnId, decidedRX);
         return new DefaultResult(minTxnId, maxTxnId, decidedRX, CloseableIterator.wrap(result.iterator()));
@@ -138,7 +140,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         private void search(int storeId, TableId tableId,
                             byte[] start, byte[] end,
                             TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX,
-                            Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                            Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             StoreIndex idx = storeIndexes.get(storeId);
             if (idx == null) return;
@@ -148,7 +150,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         private void search(int storeId, TableId tableId,
                             byte[] key,
                             TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX,
-                            Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                            Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             StoreIndex idx = storeIndexes.get(storeId);
             if (idx == null) return;
@@ -181,7 +183,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         public void search(TableId tableId,
                            byte[] start, byte[] end,
                            TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX,
-                           Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                           Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             TableIndex index = tableIndex.get(tableId);
             if (index == null) return;
@@ -191,7 +193,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         public void search(TableId tableId,
                            byte[] key,
                            TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX,
-                           Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                           Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             TableIndex index = tableIndex.get(tableId);
             if (index == null) return;
@@ -201,7 +203,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
 
     private static class TableIndex
     {
-        private final RangeTree<byte[], IndexRange, TxnId> index = createRangeTree();
+        private final RangeTree<byte[], IndexedRange, TxnId> index = createRangeTree();
         private TxnId min = TxnId.MAX;
         private TxnId max = TxnId.NONE;
         private @Nullable TxnId maxRX = TxnId.NONE;
@@ -214,7 +216,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         {
             byte[] start = OrderedRouteSerializer.serializeTokenOnly(ts.start());
             byte[] end = OrderedRouteSerializer.serializeTokenOnly(ts.end());
-            IndexRange range = new IndexRange(start, end);
+            IndexedRange range = new IndexedRange(start, end);
 
             index.add(range, id);
             if (min.compareTo(id) > 0)
@@ -230,12 +232,12 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
 
         private void search(byte[] start, byte[] end,
                            TxnId minTxnId, Timestamp maxTxnId, @Nullable DecidedRX decidedRX,
-                           Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                           Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             if (minTxnId.compareTo(max) > 0) return;
             if (maxTxnId.compareTo(min) < 0) return;
             if (maxRX != null && !RouteIndexFormat.includeByDecidedRX(decidedRX, maxRX)) return;
-            index.search(new IndexRange(start, end), e -> {
+            index.search(new IndexedRange(start, end), e -> {
                 if (minTxnId.compareTo(e.getValue()) > 0) return;
                 if (maxTxnId.compareTo(e.getValue()) < 0) return;
                 fn.accept(e);
@@ -244,7 +246,7 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
 
         private void search(byte[] key,
                             TxnId minTxnId, Timestamp maxTxnId, @Nullable MaxDecidedRX.DecidedRX decidedRX,
-                            Consumer<Map.Entry<IndexRange, TxnId>> fn)
+                            Consumer<Map.Entry<IndexedRange, TxnId>> fn)
         {
             if (minTxnId.compareTo(max) > 0) return;
             if (maxTxnId.compareTo(min) < 0) return;
@@ -257,18 +259,18 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
         }
     }
 
-    private static RangeTree<byte[], IndexRange, TxnId> createRangeTree()
+    private static RangeTree<byte[], IndexedRange, TxnId> createRangeTree()
     {
         return new RTree<>((a, b) -> ByteArrayUtil.compareUnsigned(a, 0, b, 0, a.length), new RangeTree.Accessor<>()
         {
             @Override
-            public byte[] start(IndexRange range)
+            public byte[] start(IndexedRange range)
             {
                 return range.start;
             }
 
             @Override
-            public byte[] end(IndexRange range)
+            public byte[] end(IndexedRange range)
             {
                 return range.end;
             }
@@ -282,13 +284,13 @@ public class JournalSegmentRangeSearcher<V> implements JournalRangeSearcher
             }
 
             @Override
-            public boolean intersects(IndexRange range, byte[] start, byte[] end)
+            public boolean intersects(IndexedRange range, byte[] start, byte[] end)
             {
                 return range.intersects(start, end);
             }
 
             @Override
-            public boolean intersects(IndexRange left, IndexRange right)
+            public boolean intersects(IndexedRange left, IndexedRange right)
             {
                 return left.intersects(right.start, right.end);
             }

src/java/org/apache/cassandra/service/accord/journal/TableAndSegmentKeyIterator.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import com.google.common.collect.AbstractIterator;
+
+import accord.utils.Invariants;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.journal.Journal;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.apache.cassandra.service.accord.JournalKey.SUPPORT;
+
+class TableAndSegmentKeyIterator<V> extends AbstractIterator<Journal.KeyRefs<JournalKey>> implements CloseableIterator<Journal.KeyRefs<JournalKey>>
+{
+    final Journal<JournalKey, V>.SegmentKeyIterator journalIterator;
+    final TableKeyIterator tableIterator;
+
+    TableAndSegmentKeyIterator(Journal<JournalKey, V> journal, ColumnFamilyStore table, JournalKey min, JournalKey max, boolean includeActive, long minSegment)
+    {
+        // We must initialise journal reader first, else we may race with segment->table compaction and miss some data
+        // that is, the following sequence could happen:
+        //  - Select sstables to read
+        //  - Segments compacted; segments removed and sstables added
+        //  - Segment iterator created
+        // TODO (expected): segments should be sstables on creation
+        this.journalIterator = journal.segmentKeyIterator(min, max, segment -> segment.id() >= minSegment && (includeActive || segment.isStatic()));
+        this.tableIterator = new TableKeyIterator(table, min, max, minSegment);
+    }
+
+    JournalKey prevFromTable = null;
+    JournalKey prevFromJournal = null;
+
+    @Override
+    protected Journal.KeyRefs<JournalKey> computeNext()
+    {
+        JournalKey tableKey = tableIterator.hasNext() ? tableIterator.peek() : null;
+        JournalKey journalKey = journalIterator.hasNext() ? journalIterator.peek().key() : null;
+
+        if (journalKey != null)
+        {
+            Invariants.require(prevFromJournal == null || SUPPORT.compare(journalKey, prevFromJournal) >= 0, // == for case where we have not consumed previous on prev iteration
+                               "Incorrect sort order in journal segments: %s should strictly follow %s", journalKey, prevFromJournal);
+            prevFromJournal = journalKey;
+        }
+        else
+        {
+            prevFromJournal = null;
+        }
+
+        if (tableKey != null)
+        {
+            Invariants.require(prevFromTable == null || SUPPORT.compare(tableKey, prevFromTable) >= 0, // == for case where we have not consumed previous on prev iteration
+                               "Incorrect sort order in journal table: %s should strictly follow %s", tableKey, prevFromTable);
+            prevFromTable = tableKey;
+        }
+        else
+        {
+            prevFromTable = null;
+        }
+
+        if (tableKey == null)
+            return journalKey == null ? endOfData() : journalIterator.next();
+
+        if (journalKey == null)
+            return new Journal.KeyRefs<>(tableIterator.next());
+
+        int cmp = SUPPORT.compare(tableKey, journalKey);
+        if (cmp == 0)
+        {
+            tableIterator.next();
+            return journalIterator.next();
+        }
+
+        return cmp < 0 ? new Journal.KeyRefs<>(tableIterator.next()) : journalIterator.next();
+    }
+
+    public void close()
+    {
+        tableIterator.close();
+        journalIterator.close();
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/TableKeyIterator.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.CheckForNull;
+
+import com.google.common.collect.AbstractIterator;
+
+import accord.utils.Invariants;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.utils.CloseableIterator;
+
+import static org.apache.cassandra.service.accord.AccordKeyspace.JournalColumns.getJournalKey;
+import static org.apache.cassandra.service.accord.JournalKey.SUPPORT;
+
+class TableKeyIterator extends AbstractIterator<JournalKey> implements CloseableIterator<JournalKey>
+{
+    private final UnfilteredPartitionIterator mergeIterator;
+    private final ColumnFamilyStore.RefViewFragment view;
+
+    TableKeyIterator(ColumnFamilyStore table, JournalKey min, JournalKey max, long minSegment)
+    {
+        Invariants.require((min != null && max != null) || min == max);
+        view = table.selectAndReference(View.select(SSTableSet.LIVE, r -> (max == null || SUPPORT.compare(getJournalKey(r.getFirst()), max) <= 0)
+                                                                          && (min == null || SUPPORT.compare(getJournalKey(r.getLast()), min) >= 0)
+                                                                          && (r.getSSTableMetadata().coveredClustering.end().isArtificial() || LongType.instance.compose(r.getSSTableMetadata().coveredClustering.end().bufferAt(0)) >= minSegment)
+        ));
+        List<ISSTableScanner> scanners = new ArrayList<>();
+        for (SSTableReader sstable : view.sstables)
+        {
+
+            if (min == null) scanners.add(sstable.getScanner());
+            else
+                scanners.add(sstable.getScanner(new Bounds(AccordKeyspace.JournalColumns.decorate(min), AccordKeyspace.JournalColumns.decorate(max))));
+        }
+
+        mergeIterator = view.sstables.isEmpty()
+                        ? EmptyIterators.unfilteredPartition(table.metadata())
+                        : UnfilteredPartitionIterators.merge(scanners, UnfilteredPartitionIterators.MergeListener.NOOP);
+    }
+
+    @CheckForNull
+    protected JournalKey computeNext()
+    {
+        JournalKey ret = null;
+        if (mergeIterator.hasNext())
+        {
+            try (UnfilteredRowIterator partition = mergeIterator.next())
+            {
+                ret = getJournalKey(partition.partitionKey());
+                while (partition.hasNext())
+                    partition.next();
+            }
+        }
+
+        if (ret != null)
+            return ret;
+        else
+            return endOfData();
+    }
+
+    @Override
+    public void close()
+    {
+        mergeIterator.close();
+        view.close();
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/TableRecordIterator.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.journal;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.StorageHook;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.journal.RecordConsumer;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.JournalKey;
+import org.apache.cassandra.utils.Closeable;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+import static org.apache.cassandra.io.sstable.SSTableReadsListener.NOOP_LISTENER;
+
+class TableRecordIterator implements Closeable, RecordConsumer<JournalKey>
+{
+    final JournalKey key;
+    final List<UnfilteredRowIterator> unmerged;
+    final UnfilteredRowIterator merged;
+
+    long segment;
+    int offset;
+    ByteBuffer value;
+    int userVersion;
+
+    TableRecordIterator(JournalKey key, List<UnfilteredRowIterator> unmerged, UnfilteredRowIterator merged)
+    {
+        this.key = key;
+        this.unmerged = unmerged;
+        this.merged = merged;
+    }
+
+    static TableRecordIterator all(ColumnFamilyStore cfs, JournalKey key, OpOrder.Group readOrder)
+    {
+        DecoratedKey pk = AccordKeyspace.JournalColumns.decorate(key);
+        List<UnfilteredRowIterator> iters = new ArrayList<>(3);
+        try
+        {
+            ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, pk));
+            for (SSTableReader sstable : view.sstables)
+            {
+                if (!sstable.mayContainAssumingKeyIsInRange(pk))
+                    continue;
+
+                UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs, sstable, pk, Slices.ALL, ColumnFilter.all(cfs.metadata()), false, NOOP_LISTENER);
+                if (iter.getClass() != EmptyIterators.EmptyUnfilteredRowIterator.class)
+                    iters.add(iter);
+            }
+
+            return new TableRecordIterator(key, iters, iters.isEmpty() ? null : UnfilteredRowIterators.merge(iters));
+        }
+        catch (Throwable t)
+        {
+            for (UnfilteredRowIterator iter : iters)
+            {
+                try { iter.close(); }
+                catch (Throwable t2) { t.addSuppressed(t2); }
+            }
+            throw t;
+        }
+    }
+
+    @Override
+    public void accept(long segment, int offset, JournalKey key, ByteBuffer buffer, int userVersion)
+    {
+        this.segment = segment;
+        this.offset = offset;
+        this.value = buffer;
+        this.userVersion = userVersion;
+    }
+
+    boolean advance()
+    {
+        if (merged == null || !merged.hasNext())
+            return false;
+
+        try
+        {
+            Row row = (Row) merged.next();
+            segment = LongType.instance.compose(ByteBuffer.wrap((byte[]) row.clustering().get(0)));
+            offset = Int32Type.instance.compose(ByteBuffer.wrap((byte[]) row.clustering().get(1)));
+            value = row.getCell(AccordKeyspace.JournalColumns.record).buffer();
+            userVersion = Int32Type.instance.compose(row.getCell(AccordKeyspace.JournalColumns.user_version).buffer());
+            return true;
+        }
+        catch (Throwable t)
+        {
+            throw new FSReadError("Failed to read from " + unmerged, t);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        if (merged != null)
+            merged.close();
+    }
+}

src/java/org/apache/cassandra/service/accord/journal/TopologyRecord.java

@@ -26,7 +26,9 @@ import org.agrona.collections.Int2ObjectHashMap;
 
 import accord.api.Journal;
 import accord.local.CommandStores;
+import accord.local.Node;
 import accord.primitives.Ranges;
+import accord.primitives.TxnId;
 import accord.topology.Topology;
 import accord.utils.Invariants;
 import accord.utils.UnhandledEnum;
@@ -35,198 +37,29 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.UnversionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.AccordJournalValueSerializers;
 import org.apache.cassandra.service.accord.JournalKey;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.service.accord.serializers.TopologySerializers;
-import org.apache.cassandra.service.accord.serializers.Version;
 
-public interface AccordTopologyUpdate
+import static org.apache.cassandra.service.accord.serializers.CommandStoreSerializers.rangesForEpoch;
+
+public interface TopologyRecord
 {
     Kind kind();
     void applyTo(TopologyImage accumulator);
     long epoch();
-    AccordTopologyUpdate asRepeat();
+    TopologyRecord asRepeat();
 
     Journal.TopologyUpdate getUpdate();
-    static AccordTopologyUpdate newTopology(Journal.TopologyUpdate update)
+    static TopologyRecord newTopology(Journal.TopologyUpdate update)
     {
         return new NewTopology(update);
     }
-    class RangesForEpochSerializer implements UnversionedSerializer<CommandStores.RangesForEpoch>
+
+    static JournalKey journalKey(long epoch)
     {
-        public static final RangesForEpochSerializer instance = new RangesForEpochSerializer();
-
-        @Override
-        public void serialize(CommandStores.RangesForEpoch from, DataOutputPlus out) throws IOException
-        {
-            out.writeUnsignedVInt32(from.size());
-            for (int i = 0; i < from.size(); i++)
-            {
-                out.writeLong(from.epochAtIndex(i));
-                KeySerializers.ranges.serialize(from.rangesAtIndex(i), out);
-            }
-        }
-
-        @Override
-        public CommandStores.RangesForEpoch deserialize(DataInputPlus in) throws IOException
-        {
-            int size = in.readUnsignedVInt32();
-            Ranges[] ranges = new Ranges[size];
-            long[] epochs = new long[size];
-            for (int i = 0; i < ranges.length; i++)
-            {
-                epochs[i] = in.readLong();
-                ranges[i] = KeySerializers.ranges.deserialize(in);
-            }
-            return new CommandStores.RangesForEpoch(epochs, ranges);
-        }
-
-        @Override
-        public long serializedSize(CommandStores.RangesForEpoch from)
-        {
-            long size = TypeSizes.sizeofUnsignedVInt(from.size());
-            for (int i = 0; i < from.size(); i++)
-            {
-                size += TypeSizes.LONG_SIZE;
-                size += KeySerializers.ranges.serializedSize(from.rangesAtIndex(i));
-            }
-            return size;
-        }
-    }
-
-    class TopologyUpdateSerializer implements UnversionedSerializer<Journal.TopologyUpdate>
-    {
-        public static final TopologyUpdateSerializer instance = new TopologyUpdateSerializer();
-
-        @Override
-        public void serialize(Journal.TopologyUpdate from, DataOutputPlus out) throws IOException
-        {
-            out.writeUnsignedVInt32(from.commandStores.size());
-            for (Map.Entry<Integer, CommandStores.RangesForEpoch> e : from.commandStores.entrySet())
-            {
-                out.writeUnsignedVInt32(e.getKey());
-                RangesForEpochSerializer.instance.serialize(e.getValue(), out);
-            }
-            TopologySerializers.compactTopology.serialize(from.global, out);
-        }
-
-        @Override
-        public Journal.TopologyUpdate deserialize(DataInputPlus in) throws IOException
-        {
-            int commandStoresSize = in.readUnsignedVInt32();
-            Int2ObjectHashMap<CommandStores.RangesForEpoch> commandStores = new Int2ObjectHashMap<>();
-            for (int j = 0; j < commandStoresSize; j++)
-            {
-                int commandStoreId = in.readUnsignedVInt32();
-                CommandStores.RangesForEpoch rangesForEpoch = RangesForEpochSerializer.instance.deserialize(in);
-                commandStores.put(commandStoreId, rangesForEpoch);
-            }
-            Topology global = TopologySerializers.compactTopology.deserialize(in);
-            return new Journal.TopologyUpdate(commandStores, global);
-        }
-
-        @Override
-        public long serializedSize(Journal.TopologyUpdate from)
-        {
-            long size = TypeSizes.sizeofUnsignedVInt(from.commandStores.size());
-            for (Map.Entry<Integer, CommandStores.RangesForEpoch> e : from.commandStores.entrySet())
-            {
-                size += TypeSizes.sizeofUnsignedVInt(e.getKey());
-                size += RangesForEpochSerializer.instance.serializedSize(e.getValue());
-            }
-
-            size += TopologySerializers.compactTopology.serializedSize(from.global);
-            return size;
-        }
-    }
-
-    class Serializer implements UnversionedSerializer<AccordTopologyUpdate>
-    {
-        public static Serializer instance = new Serializer();
-
-        @Override
-        public void serialize(AccordTopologyUpdate t, DataOutputPlus out) throws IOException
-        {
-            out.writeUnsignedVInt(t.epoch());
-            out.writeUnsignedVInt32(t.kind().ordinal());
-            switch (t.kind())
-            {
-                case New:
-                {
-                    TopologyUpdateSerializer.instance.serialize(((NewTopology) t).update, out);
-                    break;
-                }
-                case Repeat:
-                case Image:
-                    TopologyImage image = (TopologyImage) t;
-                    out.writeBoolean(image.update != null);
-                    if (image.update != null)
-                        TopologyUpdateSerializer.instance.serialize(image.update, out);
-                    out.writeByte(0); // defunct enum byte
-
-                    KeySerializers.ranges.serialize(image.closed, out);
-                    KeySerializers.ranges.serialize(image.retired, out);
-                    break;
-                default:
-                    throw new UnhandledEnum(t.kind());
-            }
-        }
-
-        @Override
-        public AccordTopologyUpdate deserialize(DataInputPlus in) throws IOException
-        {
-            long epoch = in.readUnsignedVInt();
-            Kind kind = Kind.values()[in.readUnsignedVInt32()];
-
-            switch (kind)
-            {
-                case New:
-                    return new NewTopology(TopologyUpdateSerializer.instance.deserialize(in));
-                case Repeat:
-                case Image:
-                    Journal.TopologyUpdate update = null;
-                    if (in.readBoolean())
-                        update = TopologyUpdateSerializer.instance.deserialize(in);
-
-                    TopologyImage image = new TopologyImage(epoch, kind, update);
-                    in.readByte(); // defunct enum byte
-
-                    image.closed = KeySerializers.ranges.deserialize(in);
-                    image.retired = KeySerializers.ranges.deserialize(in);
-                    return image;
-                default:
-                    throw new UnhandledEnum(kind);
-            }
-        }
-
-        @Override
-        public long serializedSize(AccordTopologyUpdate t)
-        {
-            long size = TypeSizes.sizeofUnsignedVInt(t.epoch());
-            size += TypeSizes.sizeofUnsignedVInt(t.kind().ordinal());
-
-            switch (t.kind())
-            {
-                case New:
-                    size += TopologyUpdateSerializer.instance.serializedSize(((NewTopology) t).update);
-                    break;
-                case Image:
-                case Repeat:
-                    TopologyImage image = (TopologyImage) t;
-                    size += TypeSizes.BOOL_SIZE;
-                    if (image.update != null)
-                        size += TopologyUpdateSerializer.instance.serializedSize(image.update);
-
-                    size += TypeSizes.BYTE_SIZE;
-                    size += KeySerializers.ranges.serializedSize(image.closed);
-                    size += KeySerializers.ranges.serializedSize(image.retired);
-                    break;
-                default:
-                    throw new UnhandledEnum(t.kind());
-            }
-            return size;
-        }
+        return new JournalKey(TxnId.fromValues(epoch, 0L, Node.Id.NONE),
+                              JournalKey.Type.TOPOLOGY_UPDATE, Integer.MAX_VALUE);
     }
 
     enum Kind
@@ -242,8 +75,7 @@ public interface AccordTopologyUpdate
         Repeat
     }
 
-
-    class TopologyImage implements AccordTopologyUpdate
+    class TopologyImage implements TopologyRecord
     {
         private final long epoch;
         private final Kind kind;
@@ -332,7 +164,7 @@ public interface AccordTopologyUpdate
         }
     }
 
-    class NewTopology implements AccordTopologyUpdate
+    class NewTopology implements TopologyRecord
     {
         public final Journal.TopologyUpdate update;
         private final long epoch;
@@ -370,7 +202,7 @@ public interface AccordTopologyUpdate
         }
 
         @Override
-        public AccordTopologyUpdate asRepeat()
+        public TopologyRecord asRepeat()
         {
             return new TopologyImage(epoch, Kind.Repeat, update);
         }
@@ -391,67 +223,137 @@ public interface AccordTopologyUpdate
         }
     }
 
-    class Accumulator implements AccordJournalValueSerializers.FlyweightImage
+    class TopologyUpdateSerializer implements UnversionedSerializer<Journal.TopologyUpdate>
     {
-        TopologyImage read, write;
+        public static final TopologyUpdateSerializer instance = new TopologyUpdateSerializer();
 
-        public Accumulator()
+        @Override
+        public void serialize(Journal.TopologyUpdate from, DataOutputPlus out) throws IOException
         {
+            out.writeUnsignedVInt32(from.commandStores.size());
+            for (Map.Entry<Integer, CommandStores.RangesForEpoch> e : from.commandStores.entrySet())
+            {
+                out.writeUnsignedVInt32(e.getKey());
+                rangesForEpoch.serialize(e.getValue(), out);
+            }
+            TopologySerializers.compactTopology.serialize(from.global, out);
         }
 
         @Override
-        public void reset(JournalKey key)
+        public Journal.TopologyUpdate deserialize(DataInputPlus in) throws IOException
         {
-            read = write = null;
+            int commandStoresSize = in.readUnsignedVInt32();
+            Int2ObjectHashMap<CommandStores.RangesForEpoch> commandStores = new Int2ObjectHashMap<>();
+            for (int j = 0; j < commandStoresSize; j++)
+            {
+                int commandStoreId = in.readUnsignedVInt32();
+                CommandStores.RangesForEpoch rfe = rangesForEpoch.deserialize(in);
+                commandStores.put(commandStoreId, rfe);
+            }
+            Topology global = TopologySerializers.compactTopology.deserialize(in);
+            return new Journal.TopologyUpdate(commandStores, global);
         }
 
-        public TopologyImage read()
+        @Override
+        public long serializedSize(Journal.TopologyUpdate from)
         {
-            return read;
-        }
+            long size = TypeSizes.sizeofUnsignedVInt(from.commandStores.size());
+            for (Map.Entry<Integer, CommandStores.RangesForEpoch> e : from.commandStores.entrySet())
+            {
+                size += TypeSizes.sizeofUnsignedVInt(e.getKey());
+                size += rangesForEpoch.serializedSize(e.getValue());
+            }
 
-        public void read(AccordTopologyUpdate update)
-        {
-            if (Objects.requireNonNull(update.kind()) == Kind.New)
-                read = new TopologyImage(update.epoch(), Kind.Image, update.getUpdate());
-            else
-                read = (TopologyImage) update;
-            write = read;
-        }
-
-        public void write(TopologyImage image)
-        {
-            Invariants.require(write == read);
-            this.write = image;
+            size += TopologySerializers.compactTopology.serializedSize(from.global);
+            return size;
         }
     }
 
-    class FlyweightSerializer implements AccordJournalValueSerializers.FlyweightSerializer<AccordTopologyUpdate, Accumulator>
+    class Serializer implements UnversionedSerializer<TopologyRecord>
     {
-        public FlyweightSerializer() {}
+        public static Serializer instance = new Serializer();
 
         @Override
-        public Accumulator mergerFor()
+        public void serialize(TopologyRecord t, DataOutputPlus out) throws IOException
         {
-            return new Accumulator();
+            out.writeUnsignedVInt(t.epoch());
+            out.writeUnsignedVInt32(t.kind().ordinal());
+            switch (t.kind())
+            {
+                case New:
+                {
+                    TopologyUpdateSerializer.instance.serialize(((NewTopology) t).update, out);
+                    break;
+                }
+                case Repeat:
+                case Image:
+                    TopologyImage image = (TopologyImage) t;
+                    out.writeBoolean(image.update != null);
+                    if (image.update != null)
+                        TopologyUpdateSerializer.instance.serialize(image.update, out);
+                    out.writeByte(0); // defunct enum byte
+
+                    KeySerializers.ranges.serialize(image.closed, out);
+                    KeySerializers.ranges.serialize(image.retired, out);
+                    break;
+                default:
+                    throw new UnhandledEnum(t.kind());
+            }
         }
 
         @Override
-        public void serialize(JournalKey key, AccordTopologyUpdate from, DataOutputPlus out, Version version) throws IOException
+        public TopologyRecord deserialize(DataInputPlus in) throws IOException
         {
-            Serializer.instance.serialize(from, out);
+            long epoch = in.readUnsignedVInt();
+            Kind kind = Kind.values()[in.readUnsignedVInt32()];
+
+            switch (kind)
+            {
+                case New:
+                    return new NewTopology(TopologyUpdateSerializer.instance.deserialize(in));
+                case Repeat:
+                case Image:
+                    Journal.TopologyUpdate update = null;
+                    if (in.readBoolean())
+                        update = TopologyUpdateSerializer.instance.deserialize(in);
+
+                    TopologyImage image = new TopologyImage(epoch, kind, update);
+                    in.readByte(); // defunct enum byte
+
+                    image.closed = KeySerializers.ranges.deserialize(in);
+                    image.retired = KeySerializers.ranges.deserialize(in);
+                    return image;
+                default:
+                    throw new UnhandledEnum(kind);
+            }
         }
 
         @Override
-        public void reserialize(JournalKey key, Accumulator from, DataOutputPlus out, Version version) throws IOException
+        public long serializedSize(TopologyRecord t)
         {
-            serialize(key, from.write, out, version);
-        }
+            long size = TypeSizes.sizeofUnsignedVInt(t.epoch());
+            size += TypeSizes.sizeofUnsignedVInt(t.kind().ordinal());
 
-        @Override
-        public void deserialize(JournalKey key, Accumulator into, DataInputPlus in, Version version) throws IOException
-        {
-            into.read(Serializer.instance.deserialize(in));
+            switch (t.kind())
+            {
+                case New:
+                    size += TopologyUpdateSerializer.instance.serializedSize(((NewTopology) t).update);
+                    break;
+                case Image:
+                case Repeat:
+                    TopologyImage image = (TopologyImage) t;
+                    size += TypeSizes.BOOL_SIZE;
+                    if (image.update != null)
+                        size += TopologyUpdateSerializer.instance.serializedSize(image.update);
+
+                    size += TypeSizes.BYTE_SIZE;
+                    size += KeySerializers.ranges.serializedSize(image.closed);
+                    size += KeySerializers.ranges.serializedSize(image.retired);
+                    break;
+                default:
+                    throw new UnhandledEnum(t.kind());
+            }
+            return size;
         }
     }
 }

src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java

@@ -39,13 +39,13 @@ import org.apache.cassandra.metrics.LatencyMetrics;
 import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.AccordEndpointMapper;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordTopology;
 import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.RequestBookkeeping;
 import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyRequestBookkeeping;
 import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.topology.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.topology.AccordTopology;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.utils.Pair;

src/java/org/apache/cassandra/service/accord/serializers/AccordSerializers.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.service.accord;
+package org.apache.cassandra.service.accord.serializers;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -45,8 +45,6 @@ import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.service.accord.serializers.IVersionedSerializer;
-import org.apache.cassandra.service.accord.serializers.Version;
 
 import static org.apache.cassandra.db.TypeSizes.sizeof;
 import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;

src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java

@@ -21,128 +21,240 @@ package org.apache.cassandra.service.accord.serializers;
 import java.io.IOException;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
 import java.util.function.IntFunction;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.api.LocalListeners.TxnListener;
 import accord.api.RoutingKey;
+import accord.impl.cfr.IdEntry;
+import accord.impl.cfr.IdMultiEntry;
+import accord.impl.cfr.IdSingleEntry;
+import accord.impl.progresslog.TxnState;
+import accord.local.CommandStores;
 import accord.local.DurableBefore;
+import accord.local.MaxConflicts;
+import accord.local.MaxDecidedRX;
 import accord.local.RedundantBefore;
+import accord.local.RejectBefore;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.SaveStatus;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
+import accord.utils.BTreeReducingRangeMap;
 import accord.utils.Invariants;
 import accord.utils.ReducingRangeMap;
+import accord.utils.VIntCoding;
+import accord.utils.btree.ReducingBTree;
 
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.UnversionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.CollectionSerializers;
-import org.apache.cassandra.utils.NullableSerializer;
+import org.apache.cassandra.utils.NoSpamLogger;
 
+import static accord.utils.Invariants.illegalState;
 import static org.apache.cassandra.service.accord.serializers.CommandSerializers.ExecuteAtSerializer.deserializeNullable;
 import static org.apache.cassandra.service.accord.serializers.CommandSerializers.ExecuteAtSerializer.serializeNullable;
 import static org.apache.cassandra.service.accord.serializers.CommandSerializers.ExecuteAtSerializer.serializedNullableSize;
 
 public class CommandStoreSerializers
 {
+    private static final Logger logger = LoggerFactory.getLogger(CommandStoreSerializers.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+    private static final int REDUCING_BTREE_MODE = 0;
+    private static final int REDUCING_ARRAY_MODE = 1;
+    private static final int REDUCING_MODE_BIT = 1;
+    private static final int REDUCING_RESERVED_FLAG_BITS = 3;
+
+    public static final UnversionedSerializer<DurableBefore> durableBefore = new DurableBeforeSerializer();
+    public static final UnversionedSerializer<MaxConflicts> maxConflicts = new MaxConflictsSerializer();
+    public static final UnversionedSerializer<MaxDecidedRX> maxDecidedRX = new ReducingRangeMapSerializer<>(new DecidedRXSerializer(), MaxDecidedRX.DecidedRX[]::new, MaxDecidedRX.SerializerSupport::create, MaxDecidedRX.EMPTY);
+    public static final UnversionedSerializer<RedundantBefore.Bounds> redundantBeforeShortBounds = new RedundantBeforeShortBoundsSerializer();
+    public static final UnversionedSerializer<RedundantBefore> redundantBefore = new ReducingRangeMapSerializer<>(redundantBeforeShortBounds, RedundantBefore.Bounds[]::new, RedundantBefore.SerializerSupport::create, RedundantBefore.EMPTY);
+    public static final UnversionedSerializer<RejectBefore> rejectBefore = new ReducingRangeMapSerializer<>(CommandSerializers.timestamp, Timestamp[]::new, RejectBefore.SerializerSupport::create, RejectBefore.EMPTY);
+    public static final UnversionedSerializer<NavigableMap<TxnId, Ranges>> bootstrapBeganAt = new TimestampToRangesMapSerializer<>(CommandSerializers.txnId);
+    public static final UnversionedSerializer<NavigableMap<Timestamp, Ranges>> safeToRead = new TimestampToRangesMapSerializer<>(CommandSerializers.timestamp);
+    public static final UnversionedSerializer<TxnListener> txnListener = new TxnListenerSerializer();
+    public static final UnversionedSerializer<TxnState> progressLogState = new ProgressLogStateSerializer();
+    public static final UnversionedSerializer<IdEntry> rangeIndexIdEntry = new RangeIndexIdEntrySerializer();
+    public static final UnversionedSerializer<CommandStores.RangesForEpoch> rangesForEpoch = new RangesForEpochSerializer();
+
     private CommandStoreSerializers() {}
 
-    public static class ReducingRangeMapSerializer<T, R extends ReducingRangeMap<T>> implements UnversionedSerializer<R>
+    // TODO (expected): use flags to switch to bitset encoding for nulls
+    private static abstract class AbstractReducingRangeMapSerializer<V, Map extends ReducingRangeMap<V>> implements UnversionedSerializer<Map>
     {
-        final UnversionedSerializer<T> valueSerializer;
-        final IntFunction<T[]> newValueArray;
-        final BiFunction<RoutingKey[], T[], R> constructor;
+        final IntFunction<V[]> newValueArray;
+        final BiFunction<RoutingKey[], V[], Map> constructor;
+        final Map empty;
 
-        public ReducingRangeMapSerializer(UnversionedSerializer<T> valueSerializer, IntFunction<T[]> newValueArray, BiFunction<RoutingKey[], T[], R> constructor)
+        public AbstractReducingRangeMapSerializer(IntFunction<V[]> newValueArray, BiFunction<RoutingKey[], V[], Map> constructor, Map empty)
         {
-            this.valueSerializer = valueSerializer;
             this.newValueArray = newValueArray;
             this.constructor = constructor;
+            this.empty = empty;
+        }
+
+        protected abstract int flags(Map map);
+        protected abstract UnversionedSerializer<V> valueSerializer(int flags);
+
+        private int safeFlags(Map map)
+        {
+            int flags = flags(map);
+            Invariants.require((flags & ((1 << REDUCING_RESERVED_FLAG_BITS) - 1)) == 0);
+            return flags | REDUCING_ARRAY_MODE;
         }
 
         @Override
-        public void serialize(R map, DataOutputPlus out) throws IOException
+        public void serialize(Map map, DataOutputPlus out) throws IOException
         {
-            out.writeBoolean(true);
+            int flags = safeFlags(map);
             int mapSize = map.size();
+            out.writeUnsignedVInt32(flags);
             out.writeUnsignedVInt32(mapSize);
 
+            if (mapSize == 0)
+                return;
+
+            UnversionedSerializer<V> valueSerializer = valueSerializer(flags);
+
             for (int i=0; i<mapSize; i++)
             {
                 KeySerializers.routingKey.serialize(map.startAt(i), out);
                 valueSerializer.serialize(map.valueAt(i), out);
             }
-            if (mapSize > 0)
-                KeySerializers.routingKey.serialize(map.startAt(mapSize), out);
+            KeySerializers.routingKey.serialize(map.startAt(mapSize), out);
         }
 
         @Override
-        public R deserialize(DataInputPlus in) throws IOException
+        public Map deserialize(DataInputPlus in) throws IOException
         {
-            in.readBoolean();
+            int flags = in.readUnsignedVInt32();
             int mapSize = in.readUnsignedVInt32();
+
+            if (mapSize == 0)
+                return empty;
+
             RoutingKey[] keys = new RoutingKey[mapSize + 1];
-            T[] values = newValueArray.apply(mapSize);
+            V[] values = newValueArray.apply(mapSize);
+            UnversionedSerializer<V> valueSerializer = valueSerializer(flags);
             for (int i=0; i<mapSize; i++)
             {
                 keys[i] = KeySerializers.routingKey.deserialize(in);
                 values[i] = valueSerializer.deserialize(in);
             }
-            if (mapSize > 0)
-                keys[mapSize] = KeySerializers.routingKey.deserialize(in);
+            keys[mapSize] = KeySerializers.routingKey.deserialize(in);
+
             return constructor.apply(keys, values);
         }
 
         @Override
-        public long serializedSize(R map)
+        public long serializedSize(Map map)
         {
-            long size = TypeSizes.BOOL_SIZE;
+            int flags = safeFlags(map);
             int mapSize = map.size();
+
+            long size = 0;
+            size += TypeSizes.sizeofUnsignedVInt(flags);
             size += TypeSizes.sizeofUnsignedVInt(mapSize);
+
+            if (mapSize == 0)
+                return size;
+
+            UnversionedSerializer<V> valueSerializer = valueSerializer(flags);
             for (int i=0; i<mapSize; i++)
             {
                 size += KeySerializers.routingKey.serializedSize(map.startAt(i));
                 size += valueSerializer.serializedSize(map.valueAt(i));
             }
-            if (mapSize > 0)
-                size += KeySerializers.routingKey.serializedSize(map.startAt(mapSize));
-
+            size += KeySerializers.routingKey.serializedSize(map.startAt(mapSize));
             return size;
         }
     }
 
-    public static UnversionedSerializer<DurableBefore> durableBefore = new ReducingRangeMapSerializer<>(NullableSerializer.wrap(new UnversionedSerializer<DurableBefore.Entry>()
+    static class ReducingRangeMapSerializer<T, Map extends ReducingRangeMap<T>> extends AbstractReducingRangeMapSerializer<T, Map> implements UnversionedSerializer<Map>
     {
-        @Override
-        public void serialize(DurableBefore.Entry t, DataOutputPlus out) throws IOException
+        final UnversionedSerializer<T> defaultValueSerializer;
+
+        public ReducingRangeMapSerializer(UnversionedSerializer<T> defaultValueSerializer, IntFunction<T[]> newValueArray, BiFunction<RoutingKey[], T[], Map> constructor, Map empty)
         {
-            CommandSerializers.txnId.serialize(t.quorumBefore, out);
-            CommandSerializers.txnId.serialize(t.universalBefore, out);
+            super(newValueArray, constructor, empty);
+            this.defaultValueSerializer = defaultValueSerializer;
         }
 
         @Override
-        public DurableBefore.Entry deserialize(DataInputPlus in) throws IOException
+        protected int flags(Map map)
         {
-            TxnId quorumBefore = CommandSerializers.txnId.deserialize(in);
-            TxnId universalBefore = CommandSerializers.txnId.deserialize(in);
-            return new DurableBefore.Entry(quorumBefore, universalBefore);
+            return 0;
         }
 
         @Override
-        public long serializedSize(DurableBefore.Entry t)
+        protected UnversionedSerializer<T> valueSerializer(int flags)
         {
-            return   CommandSerializers.txnId.serializedSize(t.quorumBefore)
-                   + CommandSerializers.txnId.serializedSize(t.universalBefore);
+            return defaultValueSerializer;
         }
-    }), DurableBefore.Entry[]::new, DurableBefore.SerializerSupport::create);
+    }
 
-    public static final UnversionedSerializer<RedundantBefore.Bounds> redundantBeforeEntry = new UnversionedSerializer<>()
+    private static final class DecidedRXSerializer implements UnversionedSerializer<MaxDecidedRX.DecidedRX>
     {
+        private DecidedRXSerializer() {}
+
+        @Override
+        public void serialize(MaxDecidedRX.DecidedRX t, DataOutputPlus out) throws IOException
+        {
+            if (t == null)
+            {
+                CommandSerializers.txnId.serialize(null, out);
+            }
+            else
+            {
+                CommandSerializers.txnId.serialize(t.any, out);
+                CommandSerializers.txnId.serialize(t.hlcBound, out);
+            }
+        }
+
+        @Override
+        public MaxDecidedRX.DecidedRX deserialize(DataInputPlus in) throws IOException
+        {
+            TxnId any = CommandSerializers.txnId.deserialize(in);
+            if (any == null)
+                return null;
+            TxnId hlcBound = CommandSerializers.txnId.deserialize(in);
+            return new MaxDecidedRX.DecidedRX(any, hlcBound);
+        }
+
+        @Override
+        public long serializedSize(MaxDecidedRX.DecidedRX t)
+        {
+            if (t == null)
+                return CommandSerializers.txnId.serializedSize(null);
+
+            return CommandSerializers.txnId.serializedSize(t.any)
+                   + CommandSerializers.txnId.serializedSize(t.hlcBound);
+        }
+    }
+
+    private static class RedundantBeforeShortBoundsSerializer implements UnversionedSerializer<RedundantBefore.Bounds>
+    {
+        private RedundantBeforeShortBoundsSerializer() {}
+
         @Override
         public void serialize(RedundantBefore.Bounds b, DataOutputPlus out) throws IOException
         {
+            // was previously wrapped in NullableSerializer; inlined logic here so we can convert to flags in future and save bytes
+            if (b == null)
+            {
+                out.writeByte(0);
+                return;
+            }
+            out.writeByte(1);
+
             KeySerializers.range.serialize(b.range, out);
             Invariants.require(b.startEpoch <= b.endEpoch);
             out.writeUnsignedVInt(b.startEpoch);
@@ -164,13 +276,16 @@ public class CommandStoreSerializers
         private short cast(long v)
         {
             if ((v & ~0xFFFF) != 0)
-                throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization version bump.");
+                throw new IllegalStateException("Cannot serialize RedundantStatus larger than 0xFFFF. Requires serialization changes.");
             return (short)v;
         }
 
         @Override
         public RedundantBefore.Bounds deserialize(DataInputPlus in) throws IOException
         {
+            if (in.readByte() == 0)
+                return null;
+
             Range range = KeySerializers.range.deserialize(in);
             long startEpoch = in.readUnsignedVInt();
             long endEpoch = in.readUnsignedVInt();
@@ -192,7 +307,10 @@ public class CommandStoreSerializers
         @Override
         public long serializedSize(RedundantBefore.Bounds b)
         {
-            long size = KeySerializers.range.serializedSize(b.range);
+            if (b == null)
+                return 1;
+
+            long size = 1 + KeySerializers.range.serializedSize(b.range);
             size += TypeSizes.sizeofUnsignedVInt(b.startEpoch);
             size += TypeSizes.sizeofUnsignedVInt(b.endEpoch == Long.MAX_VALUE ? 0 : 1 + b.endEpoch - b.startEpoch);
             size += serializedNullableSize(b.staleUntilAtLeast);
@@ -204,14 +322,13 @@ public class CommandStoreSerializers
             size += 2L * 2 * b.bounds.length;
             return size;
         }
-    };
-    public static UnversionedSerializer<RedundantBefore> redundantBefore = new ReducingRangeMapSerializer<>(NullableSerializer.wrap(redundantBeforeEntry), RedundantBefore.Bounds[]::new, RedundantBefore.SerializerSupport::create);
+    }
 
-    private static class TimestampToRangesSerializer<T extends Timestamp> implements UnversionedSerializer<NavigableMap<T, Ranges>>
+    private static class TimestampToRangesMapSerializer<T extends Timestamp> implements UnversionedSerializer<NavigableMap<T, Ranges>>
     {
         private final UnversionedSerializer<T> timestampSerializer;
 
-        public TimestampToRangesSerializer(UnversionedSerializer<T> timestampSerializer)
+        public TimestampToRangesMapSerializer(UnversionedSerializer<T> timestampSerializer)
         {
             this.timestampSerializer = timestampSerializer;
         }
@@ -236,6 +353,502 @@ public class CommandStoreSerializers
         }
     }
 
-    public static final UnversionedSerializer<NavigableMap<TxnId, Ranges>> bootstrapBeganAt = new TimestampToRangesSerializer<>(CommandSerializers.txnId);
-    public static final UnversionedSerializer<NavigableMap<Timestamp, Ranges>> safeToRead = new TimestampToRangesSerializer<>(CommandSerializers.timestamp);
+    private static abstract class BTreeReducingRangeMapSerializer<E extends ReducingBTree.Entry<E>, Map extends BTreeReducingRangeMap<E>> implements UnversionedSerializer<Map>
+    {
+        private static final int RESERVED_MAP_MASK = 0x3;
+
+        private static final int DISCONTIGUOUS = 1;
+        private static final int NEW_PREFIX = 2;
+
+        public BTreeReducingRangeMapSerializer()
+        {
+        }
+
+        abstract Map empty();
+        abstract BTreeReducingRangeMap.Builder<E, Map> builder();
+        abstract void serializeWithoutRange(E e, DataOutputPlus out) throws IOException;
+        abstract long serializedSizeWithoutRange(E e);
+        abstract E deserialize(RoutingKey start, RoutingKey end, DataInputPlus in, int mapFlags) throws IOException;
+        abstract E deserializeArrayModeWithoutRange(DataInputPlus in) throws IOException;
+
+        protected int mapFlags() { return 0; }
+
+        @Override
+        public void serialize(Map map, DataOutputPlus out) throws IOException
+        {
+            // for upgrading non-tree structures
+            int mapFlags = mapFlags();
+            Invariants.require((mapFlags & RESERVED_MAP_MASK) == 0);
+            mapFlags |= REDUCING_BTREE_MODE;
+            int mapSize = map.size();
+            out.writeUnsignedVInt32(mapFlags);
+            out.writeUnsignedVInt32(mapSize);
+
+            if (mapSize == 0)
+                return;
+
+            E prev = null;
+            int fixedLength = 0;
+            for (E e : map)
+            {
+                int flags = 0;
+                if (prev == null)
+                {
+                    flags = NEW_PREFIX | DISCONTIGUOUS;
+                }
+                else
+                {
+                    int c = prev.end().compareTo(e.start());
+                    if (c > 0)
+                        throw illegalState("Not well-formed: %s overlaps %s in %s", prev, e, map);
+
+                    if (c < 0)
+                    {
+                        flags = DISCONTIGUOUS;
+                        if (!prev.prefix().equals(e.prefix()))
+                            flags |= NEW_PREFIX;
+                    }
+                    out.writeByte(flags);
+                }
+
+                if ((flags & DISCONTIGUOUS) != 0)
+                {
+                    if ((flags & NEW_PREFIX) != 0)
+                    {
+                        KeySerializers.routingKey.serializePrefix(e.prefix(), out);
+                        fixedLength = KeySerializers.routingKey.fixedKeyLengthForPrefix(e.prefix());
+                    }
+                    if (fixedLength < 0)
+                        out.writeUnsignedVInt32(KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.start()));
+                    KeySerializers.routingKey.serializeWithoutPrefixOrLength(e.start(), out);
+                }
+                if (fixedLength < 0)
+                    out.writeUnsignedVInt32(KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.end()));
+                KeySerializers.routingKey.serializeWithoutPrefixOrLength(e.end(), out);
+                serializeWithoutRange(e, out);
+                prev = e;
+            }
+        }
+
+        @Override
+        public Map deserialize(DataInputPlus in) throws IOException
+        {
+            int mapFlags = in.readUnsignedVInt32();
+            int mapSize = in.readUnsignedVInt32();
+
+            if (mapSize == 0)
+                return empty();
+
+            try (BTreeReducingRangeMap.Builder<E, Map> builder = builder())
+            {
+                if ((mapFlags & REDUCING_MODE_BIT) == REDUCING_BTREE_MODE)
+                {
+                    Object prefix = null;
+                    RoutingKey prevEnd = null;
+                    E prev = null;
+                    int fixedLength = 0;
+                    while (mapSize-- > 0)
+                    {
+                        int flags;
+                        if (prefix == null) flags = NEW_PREFIX | DISCONTIGUOUS;
+                        else flags = in.readByte();
+
+                        RoutingKey start;
+                        if ((flags & DISCONTIGUOUS) == 0)
+                        {
+                            start = prevEnd;
+                        }
+                        else
+                        {
+                            if ((flags & NEW_PREFIX) != 0)
+                            {
+                                prefix = KeySerializers.routingKey.deserializePrefix(in);
+                                fixedLength = KeySerializers.routingKey.fixedKeyLengthForPrefix(in);
+                            }
+                            int length = fixedLength >= 0 ? fixedLength : in.readUnsignedVInt32();
+                            start = KeySerializers.routingKey.deserializeWithPrefix(prefix, length, in);
+                        }
+
+                        int length = fixedLength >= 0 ? fixedLength : in.readUnsignedVInt32();
+                        RoutingKey end = KeySerializers.routingKey.deserializeWithPrefix(prefix, length, in);
+                        E cur = deserialize(start, end, in, mapFlags);
+                        if ((flags & DISCONTIGUOUS) != 0)
+                        {
+                            if (prev != null && prev.end().compareTo(start) > 0)
+                            {
+                                if (prev.end().compareTo(end) > 0)
+                                {
+                                    noSpamLogger.warn("BTreeReducingRangeMap not well-formed: {} not before {}; skipping", prev, cur);
+                                    prevEnd = end;
+                                    continue;
+                                }
+                                else
+                                {
+                                    E newCur = cur.with(prev.end(), end);
+                                    noSpamLogger.warn("BTreeReducingRangeMap not well-formed: {} not before {}; appending {}", prev, cur, newCur);
+                                    cur = newCur;
+                                }
+                            }
+                        }
+                        builder.append(cur);
+                        prevEnd = end;
+                        prev = cur;
+                    }
+                }
+                else
+                {
+                    // read linear format for upgrading from non-tree versions of collections
+                    E prev = null;
+                    RoutingKey prevStart = null;
+                    while (mapSize-- > 0)
+                    {
+                        RoutingKey prevEnd = KeySerializers.routingKey.deserialize(in);
+                        if (prev != null)
+                            builder.append(prev.with(prevStart, prevEnd));
+                        prev = deserializeArrayModeWithoutRange(in);
+                        prevStart = prevEnd;
+                    }
+                    RoutingKey prevEnd = KeySerializers.routingKey.deserialize(in);
+                    if (prev != null)
+                        builder.append(prev.with(prevStart, prevEnd));
+
+                }
+                return builder.build();
+            }
+
+        }
+
+        @Override
+        public long serializedSize(Map map)
+        {
+            // for upgrading non-tree structures
+            // noinspection UnnecessaryLocalVariable
+            int mapFlags = REDUCING_BTREE_MODE;
+            int mapSize = map.size();
+
+            long size = TypeSizes.sizeofUnsignedVInt(mapFlags);
+            size += TypeSizes.sizeofUnsignedVInt(mapSize);
+
+            if (mapSize == 0)
+                return size;
+
+            E prev = null;
+            int fixedLength = 0;
+            for (E e : map)
+            {
+                int flags = 0;
+                if (prev == null)
+                {
+                    fixedLength = KeySerializers.routingKey.fixedKeyLengthForPrefix(e.prefix());
+                    flags = NEW_PREFIX | DISCONTIGUOUS;
+                }
+                else
+                {
+                    if (!prev.end().equals(e.start()))
+                    {
+                        flags = DISCONTIGUOUS;
+                        if (!prev.prefix().equals(e.prefix()))
+                            flags |= NEW_PREFIX;
+                    }
+                    size += 1;
+                }
+
+                if ((flags & DISCONTIGUOUS) != 0)
+                {
+                    if ((flags & NEW_PREFIX) != 0)
+                    {
+                        size += KeySerializers.routingKey.serializedSizeOfPrefix(e.prefix());
+                        fixedLength = KeySerializers.routingKey.fixedKeyLengthForPrefix(e.prefix());
+                    }
+                    if (fixedLength < 0)
+                        size += VIntCoding.sizeOfUnsignedVInt(KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.start()));
+                    size += KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.start());
+                }
+                if (fixedLength < 0)
+                    size +=  VIntCoding.sizeOfUnsignedVInt(KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.start()));
+                size += KeySerializers.routingKey.serializedSizeWithoutPrefixOrLength(e.end());
+                size += serializedSizeWithoutRange(e);
+                prev = e;
+            }
+
+            return size;
+        }
+    }
+
+    private static final class MaxConflictsSerializer extends BTreeReducingRangeMapSerializer<MaxConflicts.Entry, MaxConflicts>
+    {
+        // use top bits of a single byte vint, to leave room for base impl to fill other way
+        private static final int SEPARATE_WRITES = 0x40;
+
+        private MaxConflictsSerializer() {}
+
+        @Override
+        protected int mapFlags()
+        {
+            return SEPARATE_WRITES;
+        }
+
+        @Override
+        MaxConflicts empty()
+        {
+            return MaxConflicts.EMPTY;
+        }
+
+        @Override
+        BTreeReducingRangeMap.Builder<MaxConflicts.Entry, MaxConflicts> builder()
+        {
+            return new MaxConflicts.Builder();
+        }
+
+        @Override
+        void serializeWithoutRange(MaxConflicts.Entry entry, DataOutputPlus out) throws IOException
+        {
+            CommandSerializers.timestamp.serialize(entry.any, out);
+            CommandSerializers.timestamp.serialize(entry.write, out);
+        }
+
+        @Override
+        long serializedSizeWithoutRange(MaxConflicts.Entry entry)
+        {
+            return CommandSerializers.timestamp.serializedSize(entry.any)
+                    + CommandSerializers.timestamp.serializedSize(entry.write);
+        }
+
+        @Override
+        MaxConflicts.Entry deserialize(RoutingKey start, RoutingKey end, DataInputPlus in, int mapFlags) throws IOException
+        {
+            Timestamp all = CommandSerializers.timestamp.deserialize(in);
+            Timestamp writes = all;
+            if ((mapFlags & SEPARATE_WRITES) != 0)
+                writes = CommandSerializers.timestamp.deserialize(in);
+            return new MaxConflicts.Entry(start, end, all, writes);
+        }
+
+        @Override
+        MaxConflicts.Entry deserializeArrayModeWithoutRange(DataInputPlus in) throws IOException
+        {
+            Timestamp all = CommandSerializers.timestamp.deserialize(in);
+            return new MaxConflicts.Entry(all, all);
+        }
+    }
+
+    private static final class DurableBeforeSerializer extends BTreeReducingRangeMapSerializer<DurableBefore.Entry, DurableBefore>
+    {
+        private DurableBeforeSerializer() {}
+
+        @Override
+        DurableBefore empty()
+        {
+            return DurableBefore.EMPTY;
+        }
+
+        @Override
+        DurableBefore.Builder builder()
+        {
+            return new DurableBefore.Builder();
+        }
+
+        @Override
+        void serializeWithoutRange(DurableBefore.Entry entry, DataOutputPlus out) throws IOException
+        {
+            CommandSerializers.txnId.serialize(entry.quorum, out);
+            CommandSerializers.txnId.serialize(entry.universal, out);
+        }
+
+        @Override
+        long serializedSizeWithoutRange(DurableBefore.Entry entry)
+        {
+            return CommandSerializers.txnId.serializedSize(entry.quorum)
+                 + CommandSerializers.txnId.serializedSize(entry.universal);
+        }
+
+        @Override
+        DurableBefore.Entry deserialize(RoutingKey start, RoutingKey end, DataInputPlus in, int mapFlags) throws IOException
+        {
+            TxnId quorum = CommandSerializers.txnId.deserialize(in);
+            TxnId universal = CommandSerializers.txnId.deserialize(in);
+            return new DurableBefore.Entry(start, end, quorum, universal);
+        }
+
+        @Override
+        DurableBefore.Entry deserializeArrayModeWithoutRange(DataInputPlus in) throws IOException
+        {
+            if (!in.readBoolean())
+                return null;
+            TxnId quorum = CommandSerializers.txnId.deserialize(in);
+            TxnId universal = CommandSerializers.txnId.deserialize(in);
+            return DurableBefore.Entry.constructWithoutRange(quorum, universal);
+        }
+    }
+
+    private static final class TxnListenerSerializer implements UnversionedSerializer<TxnListener>
+    {
+        private TxnListenerSerializer() {}
+
+        @Override
+        public void serialize(TxnListener t, DataOutputPlus out) throws IOException
+        {
+            if (t == null)
+            {
+                CommandSerializers.txnId.serialize(null, out);
+            }
+            else
+            {
+                CommandSerializers.txnId.serialize(t.waiter, out);
+                CommandSerializers.txnId.serialize(t.waitingOn, out);
+                CommandSerializers.saveStatus.serialize(t.awaitingStatus, out);
+            }
+        }
+
+        @Override
+        public TxnListener deserialize(DataInputPlus in) throws IOException
+        {
+            TxnId waiter = CommandSerializers.txnId.deserialize(in);
+            if (waiter == null)
+                return null;
+            TxnId waitingOn = CommandSerializers.txnId.deserialize(in);
+            SaveStatus awaitingStatus = CommandSerializers.saveStatus.deserialize(in);
+            return new TxnListener(waiter, waitingOn, awaitingStatus);
+        }
+
+        @Override
+        public long serializedSize(TxnListener t)
+        {
+            if (t == null)
+                return CommandSerializers.txnId.serializedSize(null);
+
+            return CommandSerializers.txnId.serializedSize(t.waiter)
+                   + CommandSerializers.txnId.serializedSize(t.waitingOn)
+                   + CommandSerializers.saveStatus.serializedSize(t.awaitingStatus);
+        }
+    }
+
+    private static final class ProgressLogStateSerializer implements UnversionedSerializer<TxnState>
+    {
+        private ProgressLogStateSerializer() {}
+
+        @Override
+        public void serialize(TxnState t, DataOutputPlus out) throws IOException
+        {
+            if (t == null)
+            {
+                CommandSerializers.txnId.serialize(null, out);
+            }
+            else
+            {
+                CommandSerializers.txnId.serialize(t.txnId, out);
+                out.writeLong(t.encodedState());
+            }
+        }
+
+        @Override
+        public TxnState deserialize(DataInputPlus in) throws IOException
+        {
+            TxnId txnId = CommandSerializers.txnId.deserialize(in);
+            if (txnId == null)
+                return null;
+            long encodedState = in.readLong();
+            return TxnState.SerializationSupport.create(txnId, encodedState);
+        }
+
+        @Override
+        public long serializedSize(TxnState t)
+        {
+            if (t == null)
+                return CommandSerializers.txnId.serializedSize(null);
+
+            return CommandSerializers.txnId.serializedSize(t.txnId) + TypeSizes.LONG_SIZE;
+        }
+    }
+
+    private static final class RangeIndexIdEntrySerializer implements UnversionedSerializer<IdEntry>
+    {
+        private RangeIndexIdEntrySerializer() {}
+
+        @Override
+        public void serialize(IdEntry t, DataOutputPlus out) throws IOException
+        {
+            byte flags = (byte) ((t.getClass() == IdSingleEntry.class) ? 0 : 1);
+            out.writeByte(flags);
+            CommandSerializers.txnId.serialize(t, out);
+            out.writeUnsignedVInt32(t.encoded());
+            if (flags == 0)
+            {
+                IdSingleEntry e = (IdSingleEntry) t;
+                KeySerializers.range.serialize(e.range, out);
+            }
+            else
+            {
+                IdMultiEntry e = (IdMultiEntry) t;
+                KeySerializers.ranges.serialize(e.ranges, out);
+            }
+        }
+
+        @Override
+        public IdEntry deserialize(DataInputPlus in) throws IOException
+        {
+            byte flags = in.readByte();
+            TxnId txnId = CommandSerializers.txnId.deserialize(in);
+            int encoded = in.readUnsignedVInt32();
+            if (flags == 0)
+            {
+                Range range = KeySerializers.range.deserialize(in);
+                return IdEntry.SerializerSupport.create(txnId, encoded, range);
+            }
+            else
+            {
+                Ranges ranges = KeySerializers.ranges.deserialize(in);
+                return IdEntry.SerializerSupport.create(txnId, encoded, ranges);
+            }
+        }
+
+        @Override
+        public long serializedSize(IdEntry t)
+        {
+            return 1 + CommandSerializers.txnId.serializedSize(t)
+                   + (t.getClass() == IdSingleEntry.class ? KeySerializers.range.serializedSize(((IdSingleEntry)t).range)
+                                                          : KeySerializers.ranges.serializedSize(((IdMultiEntry)t).ranges));
+        }
+    }
+
+    static class RangesForEpochSerializer implements UnversionedSerializer<CommandStores.RangesForEpoch>
+    {
+        @Override
+        public void serialize(CommandStores.RangesForEpoch from, DataOutputPlus out) throws IOException
+        {
+            out.writeUnsignedVInt32(from.size());
+            for (int i = 0; i < from.size(); i++)
+            {
+                out.writeLong(from.epochAtIndex(i));
+                KeySerializers.ranges.serialize(from.rangesAtIndex(i), out);
+            }
+        }
+
+        @Override
+        public CommandStores.RangesForEpoch deserialize(DataInputPlus in) throws IOException
+        {
+            int size = in.readUnsignedVInt32();
+            Ranges[] ranges = new Ranges[size];
+            long[] epochs = new long[size];
+            for (int i = 0; i < ranges.length; i++)
+            {
+                epochs[i] = in.readLong();
+                ranges[i] = KeySerializers.ranges.deserialize(in);
+            }
+            return new CommandStores.RangesForEpoch(epochs, ranges);
+        }
+
+        @Override
+        public long serializedSize(CommandStores.RangesForEpoch from)
+        {
+            long size = TypeSizes.sizeofUnsignedVInt(from.size());
+            for (int i = 0; i < from.size(); i++)
+            {
+                size += TypeSizes.LONG_SIZE;
+                size += KeySerializers.ranges.serializedSize(from.rangesAtIndex(i));
+            }
+            return size;
+        }
+    }
+
 }

src/java/org/apache/cassandra/service/accord/serializers/EncodeAsVInt32.java

@@ -54,6 +54,12 @@ public abstract class EncodeAsVInt32<T> implements UnversionedSerializer<T>
         return withNulls(Enum::ordinal, i -> values[i]);
     }
 
+    public static <E extends Enum<?>> EncodeAsVInt32<E> withoutNulls(Class<E> clazz)
+    {
+        E[] values = clazz.getEnumConstants();
+        return withoutNulls(Enum::ordinal, i -> values[i]);
+    }
+
     static class WithNulls<T> extends EncodeAsVInt32<T>
     {
         private WithNulls(ToIntFunction<? super T> encode, IntFunction<? extends T> decode)

src/java/org/apache/cassandra/service/accord/serializers/GetDurableBeforeSerializers.java

@@ -28,7 +28,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 
 public class GetDurableBeforeSerializers
 {
-    public static final UnversionedSerializer<GetDurableBefore> request = new UnversionedSerializer<GetDurableBefore>()
+    public static final UnversionedSerializer<GetDurableBefore> request = new UnversionedSerializer<>()
     {
         @Override
         public void serialize(GetDurableBefore msg, DataOutputPlus out) throws IOException

src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java

@@ -914,7 +914,7 @@ public class KeySerializers
         @Override
         final int serializedSizeWithoutPrefix(RoutingKey routable)
         {
-            return routingKey.serializedSizeWithoutPrefix(routable);
+            return routingKey.serializedSizeWithoutPrefixOrLength(routable);
         }
 
         @Override
@@ -1067,8 +1067,8 @@ public class KeySerializers
         @Override
         final int serializedSizeWithoutPrefix(Range range)
         {
-            return routingKey.serializedSizeWithoutPrefix(range.start())
-                   + routingKey.serializedSizeWithoutPrefix(range.end());
+            return routingKey.serializedSizeWithoutPrefixOrLength(range.start())
+                   + routingKey.serializedSizeWithoutPrefixOrLength(range.end());
         }
 
         @Override
@@ -1085,9 +1085,9 @@ public class KeySerializers
             for (int i = startIndex; i < endIndex; ++i)
             {
                 Range r = ranges[i];
-                endOffset += routingKey.serializedSizeWithoutPrefix(r.start());
+                endOffset += routingKey.serializedSizeWithoutPrefixOrLength(r.start());
                 out.writeInt(endOffset);
-                endOffset += routingKey.serializedSizeWithoutPrefix(r.end());
+                endOffset += routingKey.serializedSizeWithoutPrefixOrLength(r.end());
                 out.writeInt(endOffset);
             }
         }