Fast single-partition "Ephemeral Reads"

Introduce a special kind of non-durable read that provides only per-key linearizable isolation; i.e. strict-serializable isolation for single partition-key reads.
This read creates only a happens-before edge, by collecting dependencies for execution and ensuring that execution happens strictly after these dependencies
have executed, but at no precise time otherwise. So later writes may be witnessed, and if multiple keys are read they may represents different points in time.

patch by Benedict; reviewed by Ariel Weisberg for CASSANDRA-19305

Refactor CommandsForKey for efficiency, and to support transitive dependency elision

patch by Benedict; reviewed by Aleksey Yeshchenko for CASSANDRA-19310

modules/accord

@@ -1 +1 @@
-Subproject commit 3789c5bfec50eb96157c0a55af77f78ee0cac804
+Subproject commit 6b8bef48e5780aefda6bd1ff29a6290e56ede438

src/java/org/apache/cassandra/config/AccordSpec.java

@@ -44,4 +44,9 @@ public class AccordSpec
     public DurationSpec.IntMillisecondsBound range_barrier_timeout = new DurationSpec.IntMillisecondsBound("2m");
 
     public volatile DurationSpec fast_path_update_delay = new DurationSpec.IntSecondsBound(5);
+
+    public volatile DurationSpec schedule_durability_frequency = new DurationSpec.IntSecondsBound(15);
+    public volatile DurationSpec durability_txnid_lag = new DurationSpec.IntSecondsBound(5);
+    public volatile DurationSpec shard_durability_cycle = new DurationSpec.IntMinutesBound(2);
+    public volatile DurationSpec global_durability_cycle = new DurationSpec.IntMinutesBound(10);
 }

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -5318,9 +5318,49 @@ public class DatabaseDescriptor
         return conf.accord.fast_path_update_delay.to(TimeUnit.MILLISECONDS);
     }
 
-    public static void setAccordFastPathUpdateDelayMillis(long millis)
+    public static void setAccordFastPathUpdateDelaySeconds(long seconds)
     {
-        conf.accord.fast_path_update_delay = new DurationSpec.IntMillisecondsBound(millis);
+        conf.accord.fast_path_update_delay = new DurationSpec.IntSecondsBound(seconds);
+    }
+
+    public static long getAccordScheduleDurabilityFrequency(TimeUnit unit)
+    {
+        return conf.accord.schedule_durability_frequency.to(unit);
+    }
+
+    public static void setAccordScheduleDurabilityFrequencySeconds(long seconds)
+    {
+        conf.accord.schedule_durability_frequency = new DurationSpec.IntSecondsBound(seconds);
+    }
+
+    public static long getAccordScheduleDurabilityTxnIdLag(TimeUnit unit)
+    {
+        return conf.accord.durability_txnid_lag.to(unit);
+    }
+
+    public static void setAccordScheduleDurabilityTxnIdLagSeconds(long seconds)
+    {
+        conf.accord.durability_txnid_lag = new DurationSpec.IntSecondsBound(seconds);
+    }
+
+    public static long getAccordGlobalDurabilityCycle(TimeUnit unit)
+    {
+        return conf.accord.global_durability_cycle.to(unit);
+    }
+
+    public static void setAccordGlobalDurabilityCycleSeconds(long seconds)
+    {
+        conf.accord.global_durability_cycle = new DurationSpec.IntSecondsBound(seconds);
+    }
+
+    public static long getAccordShardDurabilityCycle(TimeUnit unit)
+    {
+        return conf.accord.shard_durability_cycle.to(unit);
+    }
+
+    public static void setAccordShardDurabilityCycleSeconds(long seconds)
+    {
+        conf.accord.shard_durability_cycle = new DurationSpec.IntSecondsBound(seconds);
     }
 
     public static boolean getForceNewPreparedStatementBehaviour()

src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java

@@ -81,6 +81,10 @@ import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static accord.primitives.Txn.Kind.EphemeralRead;
+import static accord.primitives.Txn.Kind.Read;
+import static org.apache.cassandra.config.Config.NonSerialWriteStrategy.accord;
+import static org.apache.cassandra.config.DatabaseDescriptor.getNonSerialWriteStrategy;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -322,7 +326,8 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
             List<TxnNamedRead> reads = createNamedReads(options, state, ImmutableMap.of(), keySet::add);
             Keys txnKeys = toKeys(keySet);
             TxnRead read = createTxnRead(reads, txnKeys, null);
-            return new Txn.InMemory(txnKeys, read, TxnQuery.ALL);
+            Txn.Kind kind = txnKeys.size() == 1 && getNonSerialWriteStrategy() == accord ? EphemeralRead : Read;
+            return new Txn.InMemory(kind, txnKeys, read, TxnQuery.ALL, null);
         }
         else
         {

src/java/org/apache/cassandra/db/DiskBoundaryManager.java

@@ -195,7 +195,7 @@ public class DiskBoundaryManager
         List<PartitionPosition> diskBoundaries = new ArrayList<>();
         for (int i = 0; i < boundaries.size() - 1; i++)
             diskBoundaries.add(boundaries.get(i).maxKeyBound());
-        diskBoundaries.add(partitioner.getMaximumToken().maxKeyBound());
+        diskBoundaries.add(partitioner.getMaximumTokenForSplitting().maxKeyBound());
         return diskBoundaries;
     }
 }

src/java/org/apache/cassandra/db/compaction/CompactionIterator.java

@@ -98,10 +98,10 @@ import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
 import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
 import static org.apache.cassandra.service.accord.AccordKeyspace.CommandRows.maybeDropTruncatedCommandColumns;
 import static org.apache.cassandra.service.accord.AccordKeyspace.CommandRows.truncatedApply;
+import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeysAccessor;
 import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_executed_micros;
 import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_executed_timestamp;
 import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_write_timestamp;
-import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.max_timestamp;
 import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyRows.truncateTimestampsForKeyRow;
 import static org.apache.cassandra.service.accord.AccordKeyspace.deserializeDurabilityOrNull;
 import static org.apache.cassandra.service.accord.AccordKeyspace.deserializeRouteOrNull;
@@ -221,11 +221,8 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         if (isAccordTimestampsForKey(cfs))
             return new AccordTimestampsForKeyPurger(accordService);
 
-        if (isAccordDepsCommandsForKey(cfs))
-            return new AccordCommandsForKeyPurger(AccordKeyspace.DepsCommandsForKeysAccessor, accordService);
-
-        if (isAccordAllCommandsForKey(cfs))
-            return new AccordCommandsForKeyPurger(AccordKeyspace.AllCommandsForKeysAccessor, accordService);
+        if (isAccordCommandsForKey(cfs))
+            return new AccordCommandsForKeyPurger(AccordKeyspace.CommandsForKeysAccessor, accordService);
 
         throw new IllegalArgumentException("Unhandled accord table: " + cfs.keyspace.getName() + '.' + cfs.name);
     }
@@ -929,22 +926,14 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
                 lastWriteCell = null;
             }
 
-            Cell<?> maxTimestampCell = row.getCell(max_timestamp);
-            Timestamp max_timestamp = deserializeTimestampOrNull(maxTimestampCell);
-            if (max_timestamp != null && max_timestamp.compareTo(redundantBeforeTxnId) < 0)
-            {
-                maxTimestampCell = null;
-            }
-
             // No need to emit a tombstone as earlier versions of the row will also be nulled out
             // when compacted later or loaded into a commands for key
             if (lastExecuteMicrosCell == null &&
                 lastExecuteCell == null &&
-                lastWriteCell == null &&
-                maxTimestampCell == null)
+                lastWriteCell == null)
                 return null;
 
-            return truncateTimestampsForKeyRow(nowInSec, row, lastExecuteMicrosCell, lastExecuteCell, lastWriteCell, maxTimestampCell);
+            return truncateTimestampsForKeyRow(nowInSec, row, lastExecuteMicrosCell, lastExecuteCell, lastWriteCell);
         }
 
         @Override
@@ -990,11 +979,10 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
                 return row;
 
             TxnId redundantBeforeTxnId = redundantBeforeEntry.shardRedundantBefore();
-            Timestamp timestamp = accessor.getTimestamp(row);
-            if (timestamp != null && timestamp.compareTo(redundantBeforeTxnId) < 0)
-                return null;
+            if (redundantBeforeTxnId.equals(TxnId.NONE))
+                return row;
 
-            return row;
+            return CommandsForKeysAccessor.withoutRedundantCommands(partitionKey, row, redundantBeforeTxnId);
         }
 
         @Override
@@ -1050,8 +1038,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         return cfs.getKeyspaceName().equals(SchemaConstants.ACCORD_KEYSPACE_NAME) &&
                ImmutableSet.of(AccordKeyspace.COMMANDS,
                                AccordKeyspace.TIMESTAMPS_FOR_KEY,
-                               AccordKeyspace.DEPS_COMMANDS_FOR_KEY,
-                               AccordKeyspace.ALL_COMMANDS_FOR_KEY)
+                               AccordKeyspace.COMMANDS_FOR_KEY)
                            .contains(cfs.getTableName());
     }
 
@@ -1070,13 +1057,8 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
         return isAccordTable(cfs, AccordKeyspace.TIMESTAMPS_FOR_KEY);
     }
 
-    private static boolean isAccordDepsCommandsForKey(ColumnFamilyStore cfs)
+    private static boolean isAccordCommandsForKey(ColumnFamilyStore cfs)
     {
-        return isAccordTable(cfs, AccordKeyspace.DEPS_COMMANDS_FOR_KEY);
-    }
-
-    private static boolean isAccordAllCommandsForKey(ColumnFamilyStore cfs)
-    {
-        return isAccordTable(cfs, AccordKeyspace.ALL_COMMANDS_FOR_KEY);
+        return isAccordTable(cfs, AccordKeyspace.COMMANDS_FOR_KEY);
     }
 }

src/java/org/apache/cassandra/db/virtual/PartitionKeyStatsTable.java

@@ -284,7 +284,7 @@ public class PartitionKeyStatsTable implements VirtualTable
     {
         Slices s = clusteringIndexFilter.getSlices(target);
         Token startToken = target.partitioner.getMinimumToken();
-        Token endToken = target.partitioner.getMaximumToken();
+        Token endToken = target.partitioner.getMaximumTokenForSplitting();
         BigInteger startTokenValue = new BigInteger(endToken.getTokenValue().toString(), 10);
         BigInteger endTokenValue = new BigInteger(startToken.getTokenValue().toString(), 10);
 

src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java

@@ -144,6 +144,12 @@ public class ByteOrderedPartitioner implements IPartitioner
 
         @Override
         public Token nextValidToken()
+        {
+            throw new UnsupportedOperationException(String.format("Token type %s does not support token allocation.",
+                                                                  getClass().getSimpleName()));
+        }
+
+        public Token increaseSlightly()
         {
             // find first byte we can increment
             int i = token.length - 1;
@@ -180,8 +186,6 @@ public class ByteOrderedPartitioner implements IPartitioner
             if (i == -1)
             {
                 byte[] newToken = Arrays.copyOf(token, token.length - 1);
-                if (newToken.length > 0)
-                    newToken[newToken.length - 1] = (byte)-1;
                 return new BytesToken(newToken);
             }
 

src/java/org/apache/cassandra/dht/IPartitioner.java

@@ -68,9 +68,9 @@ public interface IPartitioner
      * The biggest token for this partitioner, unlike getMinimumToken, this token is actually used and users wanting to
      * include all tokens need to do getMaximumToken().maxKeyBound()
      *
-     * Not implemented for the ordered partitioners
+     * THIS IS NOT SAFE FOR PURPOSES BESIDES SPLITTING/BALANCING
      */
-    default Token getMaximumToken()
+    default Token getMaximumTokenForSplitting()
     {
         throw new UnsupportedOperationException("If you are using a splitting partitioner, getMaximumToken has to be implemented");
     }

src/java/org/apache/cassandra/dht/Murmur3Partitioner.java

@@ -468,7 +468,7 @@ public class Murmur3Partitioner implements IPartitioner
         return LongType.instance;
     }
 
-    public Token getMaximumToken()
+    public Token getMaximumTokenForSplitting()
     {
         return new LongToken(Long.MAX_VALUE);
     }

src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java

@@ -140,7 +140,7 @@ public class OrderPreservingPartitioner implements IPartitioner
         return MINIMUM;
     }
 
-    public StringToken getMaximumToken()
+    public StringToken getMaximumTokenForSplitting()
     {
         return MAXIMUM;
     }

src/java/org/apache/cassandra/dht/RandomPartitioner.java

@@ -374,7 +374,7 @@ public class RandomPartitioner implements IPartitioner
         return ownerships;
     }
 
-    public Token getMaximumToken()
+    public Token getMaximumTokenForSplitting()
     {
         return new BigIntegerToken(MAXIMUM);
     }

src/java/org/apache/cassandra/dht/Splitter.java

@@ -50,7 +50,7 @@ public abstract class Splitter extends AccordSplitter
     {
         //full range case
         if (range.left.equals(range.right))
-            return tokensInRange(new Range(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+            return tokensInRange(new Range(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting()));
 
         BigInteger totalTokens = BigInteger.ZERO;
         for (Range<Token> unwrapped : range.unwrap())
@@ -95,7 +95,7 @@ public abstract class Splitter extends AccordSplitter
     {
         //full range case
         if (range.left.equals(range.right))
-            return positionInRange(token, new Range(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+            return positionInRange(token, new Range(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting()));
 
         // leftmost token means we are on position 0.0
         if (token.equals(range.left))
@@ -115,7 +115,7 @@ public abstract class Splitter extends AccordSplitter
     public List<Token> splitOwnedRanges(int parts, List<WeightedRange> weightedRanges, boolean dontSplitRanges)
     {
         if (weightedRanges.isEmpty() || parts == 1)
-            return Collections.singletonList(partitioner.getMaximumToken());
+            return Collections.singletonList(partitioner.getMaximumTokenForSplitting());
 
         BigInteger totalTokens = BigInteger.ZERO;
         for (WeightedRange weightedRange : weightedRanges)
@@ -126,7 +126,7 @@ public abstract class Splitter extends AccordSplitter
         BigInteger perPart = totalTokens.divide(BigInteger.valueOf(parts));
         // the range owned is so tiny we can't split it:
         if (perPart.equals(BigInteger.ZERO))
-            return Collections.singletonList(partitioner.getMaximumToken());
+            return Collections.singletonList(partitioner.getMaximumTokenForSplitting());
 
         if (dontSplitRanges)
             return splitOwnedRangesNoPartialRanges(weightedRanges, perPart, parts);
@@ -155,7 +155,7 @@ public abstract class Splitter extends AccordSplitter
             }
             sum = sum.add(currentRangeWidth);
         }
-        boundaries.set(boundaries.size() - 1, partitioner.getMaximumToken());
+        boundaries.set(boundaries.size() - 1, partitioner.getMaximumTokenForSplitting());
 
         assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + weightedRanges;
         return boundaries;
@@ -192,7 +192,7 @@ public abstract class Splitter extends AccordSplitter
             }
             i++;
         }
-        boundaries.add(partitioner.getMaximumToken());
+        boundaries.add(partitioner.getMaximumTokenForSplitting());
         return boundaries;
     }
 
@@ -202,7 +202,7 @@ public abstract class Splitter extends AccordSplitter
      */
     private Token token(Token t)
     {
-        return t.equals(partitioner.getMinimumToken()) ? partitioner.getMaximumToken() : t;
+        return t.equals(partitioner.getMinimumToken()) ? partitioner.getMaximumTokenForSplitting() : t;
     }
 
     /**

src/java/org/apache/cassandra/dht/Token.java

@@ -265,7 +265,7 @@ public abstract class Token implements RingPosition<Token>, Serializable
     public Token increaseSlightly() { return nextValidToken(); }
 
     /**
-     * Returns a token that is slightly less than this.
+     * Returns a token that is slightly less than this. This is NOT guaranteed to be the directly preceding token.
      */
     abstract public Token decreaseSlightly();
 

src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java

@@ -130,7 +130,7 @@ public class BtiTableReader extends SSTableReaderWithFilter
     {
         return PartitionIterator.create(partitionIndex, metadata().partitioner, rowIndexFile, dfile,
                                         key, -1,
-                                        metadata().partitioner.getMaximumToken().maxKeyBound(), 0,
+                                        metadata().partitioner.getMaximumTokenForSplitting().maxKeyBound(), 0,
                                         descriptor.version);
     }
 

src/java/org/apache/cassandra/net/Verb.java

@@ -93,6 +93,8 @@ import org.apache.cassandra.service.accord.serializers.CommitSerializers;
 import org.apache.cassandra.service.accord.serializers.EnumSerializer;
 import org.apache.cassandra.service.accord.serializers.FetchSerializers;
 import org.apache.cassandra.service.accord.serializers.GetDepsSerializers;
+import org.apache.cassandra.service.accord.serializers.GetEphmrlReadDepsSerializers;
+import org.apache.cassandra.service.accord.serializers.GetMaxConflictSerializers;
 import org.apache.cassandra.service.accord.serializers.InformDurableSerializers;
 import org.apache.cassandra.service.accord.serializers.InformHomeDurableSerializers;
 import org.apache.cassandra.service.accord.serializers.InformOfTxnIdSerializers;
@@ -331,6 +333,10 @@ public enum Verb
     ACCORD_CHECK_STATUS_REQ         (142, P2, writeTimeout, IMMEDIATE,          () -> CheckStatusSerializers.request,       AccordService::verbHandlerOrNoop, ACCORD_CHECK_STATUS_RSP                   ),
     ACCORD_GET_DEPS_RSP             (143, P2, writeTimeout, REQUEST_RESPONSE,   () -> GetDepsSerializers.reply,             RESPONSE_HANDLER                                                            ),
     ACCORD_GET_DEPS_REQ             (144, P2, writeTimeout, IMMEDIATE,          () -> GetDepsSerializers.request,           AccordService::verbHandlerOrNoop, ACCORD_GET_DEPS_RSP                       ),
+    ACCORD_GET_EPHMRL_READ_DEPS_RSP (161, P2, writeTimeout, REQUEST_RESPONSE,   () -> GetEphmrlReadDepsSerializers.reply,   RESPONSE_HANDLER                                                            ),
+    ACCORD_GET_EPHMRL_READ_DEPS_REQ (162, P2, writeTimeout, IMMEDIATE,          () -> GetEphmrlReadDepsSerializers.request, AccordService::verbHandlerOrNoop, ACCORD_GET_EPHMRL_READ_DEPS_RSP),
+    ACCORD_GET_MAX_CONFLICT_RSP     (163, P2, writeTimeout, REQUEST_RESPONSE,   () -> GetMaxConflictSerializers.reply,      RESPONSE_HANDLER                                                            ),
+    ACCORD_GET_MAX_CONFLICT_REQ     (164, P2, writeTimeout, IMMEDIATE,          () -> GetMaxConflictSerializers.request,    AccordService::verbHandlerOrNoop, ACCORD_GET_MAX_CONFLICT_RSP),
     ACCORD_FETCH_DATA_RSP           (145, P2, repairTimeout,REQUEST_RESPONSE,   () -> FetchSerializers.reply,               RESPONSE_HANDLER                                                            ),
     ACCORD_FETCH_DATA_REQ           (146, P2, repairTimeout,IMMEDIATE,          () -> FetchSerializers.request,             AccordService::verbHandlerOrNoop, ACCORD_FETCH_DATA_RSP                     ),
     ACCORD_SET_SHARD_DURABLE_REQ    (147, P2, writeTimeout, IMMEDIATE,          () -> SetDurableSerializers.shardDurable,   AccordService::verbHandlerOrNoop, ACCORD_SIMPLE_RSP                         ),
@@ -340,7 +346,7 @@ public enum Verb
 
     ACCORD_SYNC_NOTIFY_REQ          (151, P2, writeTimeout, IMMEDIATE,          () -> Notification.listSerializer,          () -> AccordSyncPropagator.verbHandler,       ACCORD_SIMPLE_RSP             ),
 
-    ACCORD_APPLY_AND_WAIT_UNTIL_APPLIED_REQ(152, P2, writeTimeout, IMMEDIATE,   () -> ReadDataSerializers.readData,() -> AccordSyncPropagator.verbHandler,      ACCORD_READ_RSP),
+    ACCORD_APPLY_AND_WAIT_REQ       (152, P2, writeTimeout, IMMEDIATE,          () -> ReadDataSerializers.readData,         () -> AccordService.instance().verbHandler(), ACCORD_READ_RSP),
 
     CONSENSUS_KEY_MIGRATION         (153, P1, writeTimeout,  MUTATION,          () -> ConsensusKeyMigrationFinished.serializer,() -> ConsensusKeyMigrationState.consensusKeyMigrationFinishedHandler),
 

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory;
 
 import accord.primitives.Keys;
 import accord.primitives.Txn;
+import accord.utils.Invariants;
 import org.apache.cassandra.batchlog.Batch;
 import org.apache.cassandra.batchlog.BatchlogManager;
 import org.apache.cassandra.concurrent.DebuggableTask.RunnableDebuggableTask;
@@ -168,10 +169,14 @@ import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
+import static accord.primitives.Txn.Kind.EphemeralRead;
+import static accord.primitives.Txn.Kind.Read;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.Iterables.concat;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.Config.NonSerialWriteStrategy.accord;
+import static org.apache.cassandra.config.DatabaseDescriptor.getNonSerialWriteStrategy;
 import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
@@ -1218,7 +1223,7 @@ public class StorageProxy implements StorageProxyMBean
         long size = IMutation.dataSize(mutations);
         writeMetrics.mutationSize.update(size);
         writeMetricsForLevel(consistencyLevel).mutationSize.update(size);
-        NonSerialWriteStrategy nonSerialWriteStrategy = DatabaseDescriptor.getNonSerialWriteStrategy();
+        NonSerialWriteStrategy nonSerialWriteStrategy = getNonSerialWriteStrategy();
         if (nonSerialWriteStrategy.writesThroughAccord && !SchemaConstants.getSystemKeyspaces().contains(keyspaceName))
             mutateWithAccord(augmented != null ? augmented : mutations, consistencyLevel, requestTime, nonSerialWriteStrategy);
         else if (augmented != null)
@@ -1997,9 +2002,11 @@ public class StorageProxy implements StorageProxyMBean
         SinglePartitionReadCommand readCommand = group.queries.get(0);
         // If the non-SERIAL write strategy is sending all writes through Accord there is no need to use the supplied consistency
         // level since Accord will manage reading safely
-        consistencyLevel = DatabaseDescriptor.getNonSerialWriteStrategy().readCLForStrategy(consistencyLevel);
+        NonSerialWriteStrategy nonSerialWriteStrategy = getNonSerialWriteStrategy();
+        consistencyLevel = nonSerialWriteStrategy.readCLForStrategy(consistencyLevel);
         TxnRead read = TxnRead.createSerialRead(readCommand, consistencyLevel);
-        Txn txn = new Txn.InMemory(read.keys(), read, TxnQuery.ALL);
+        Invariants.checkState(read.keys().size() == 1, "Ephemeral reads are only strict-serializable for single partition reads");
+        Txn txn = new Txn.InMemory(nonSerialWriteStrategy == accord ? EphemeralRead : Read, read.keys(), read, TxnQuery.ALL, null);
         IAccordService accordService = AccordService.instance();
         accordService.maybeConvertTablesToAccord(txn);
         TxnResult txnResult = accordService.coordinate(txn, consistencyLevel, requestTime);

src/java/org/apache/cassandra/service/accord/AccordCachingState.java

@@ -121,11 +121,6 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
         return status().isComplete();
     }
 
-    public boolean canEvict()
-    {
-        return true;
-    }
-
     int estimatedSizeOnHeap(ToLongFunction<V> estimator)
     {
         shouldUpdateSize = false;

src/java/org/apache/cassandra/service/accord/AccordCommandStore.java

@@ -27,9 +27,9 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.Predicate;
 
 import javax.annotation.Nullable;
 
@@ -43,8 +43,7 @@ import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.impl.CommandsForKey;
-import accord.impl.DomainCommands;
-import accord.impl.DomainTimestamps;
+import accord.impl.CommandsSummary;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -70,7 +69,6 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.ReducingRangeMap;
-import accord.utils.TriFunction;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import accord.utils.async.Observable;
@@ -114,13 +112,10 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     private final AccordJournal journal;
     private final ExecutorService executor;
     private final ExecutionOrder executionOrder;
-    private final AccordCommandsForKeys keyCoordinator;
     private final AccordStateCache stateCache;
     private final AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache;
     private final AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache;
-    private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> depsCommandsForKeyCache;
-    private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> allCommandsForKeyCache;
-    private final AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updatesForKeyCache;
+    private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache;
     private AsyncOperation<?> currentOperation = null;
     private AccordSafeCommandStore current = null;
     private long lastSystemTimestampMicros = Long.MIN_VALUE;
@@ -153,7 +148,6 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         super(id, time, agent, dataStore, progressLogFactory, epochUpdateHolder);
         this.journal = journal;
         loggingId = String.format("[%s]", id);
-        keyCoordinator = new AccordCommandsForKeys(this);
         executor = executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + id + ']');
         executionOrder = new ExecutionOrder();
         threadId = getThreadId(executor);
@@ -174,33 +168,15 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                                 this::saveTimestampsForKey,
                                 this::validateTimestampsForKey,
                                 AccordObjectSizes::timestampsForKey);
-        depsCommandsForKeyCache =
+        commandsForKeyCache =
             stateCache.instance(RoutableKey.class,
                                 AccordSafeCommandsForKey.class,
                                 AccordSafeCommandsForKey::new,
-                                this::loadDepsCommandsForKey,
+                                this::loadCommandsForKey,
                                 this::saveCommandsForKey,
-                                this::validateDepsCommandsForKey,
+                                this::validateCommandsForKey,
                                 AccordObjectSizes::commandsForKey,
-                                keyCoordinator::createDepsCommandsNode);
-        allCommandsForKeyCache =
-            stateCache.instance(RoutableKey.class,
-                                AccordSafeCommandsForKey.class,
-                                AccordSafeCommandsForKey::new,
-                                this::loadAllCommandsForKey,
-                                this::saveCommandsForKey,
-                                this::validateAllCommandsForKey,
-                                AccordObjectSizes::commandsForKey,
-                                keyCoordinator::createDepsCommandsNode);
-        updatesForKeyCache =
-            stateCache.instance(RoutableKey.class,
-                                AccordSafeCommandsForKeyUpdate.class,
-                                AccordSafeCommandsForKeyUpdate::new,
-                                this::loadCommandsForKeyUpdate,
-                                this::saveCommandsForKeyUpdate,
-                                (key, evicting) -> true,
-                                CommandsForKeyUpdate::estimatedSizeOnHeap,
-                                keyCoordinator::createUpdatesNode);
+                                AccordCachingState::new);
 
         AccordKeyspace.loadCommandStoreMetadata(id, ((rejectBefore, durableBefore, redundantBefore, bootstrapBeganAt, safeToRead) -> {
             executor.submit(() -> {
@@ -340,19 +316,9 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         return timestampsForKeyCache;
     }
 
-    public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> depsCommandsForKeyCache()
+    public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache()
     {
-        return depsCommandsForKeyCache;
-    }
-
-    public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> allCommandsForKeyCache()
-    {
-        return allCommandsForKeyCache;
-    }
-
-    public AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updatesForKeyCache()
-    {
-        return updatesForKeyCache;
+        return commandsForKeyCache;
     }
 
     Command loadCommand(TxnId txnId)
@@ -384,39 +350,17 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         return AccordKeyspace.loadTimestampsForKey(this, (PartitionKey) key);
     }
 
-    CommandsForKey loadDepsCommandsForKey(RoutableKey key)
+    CommandsForKey loadCommandsForKey(RoutableKey key)
     {
-        return AccordKeyspace.loadDepsCommandsForKey(this, (PartitionKey) key);
+        return AccordKeyspace.loadCommandsForKey(this, (PartitionKey) key);
     }
 
-    CommandsForKey loadAllCommandsForKey(RoutableKey key)
+    boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting)
     {
-        return AccordKeyspace.loadAllCommandsForKey(this, (PartitionKey) key);
-    }
-
-    CommandsForKeyUpdate loadCommandsForKeyUpdate(RoutableKey key)
-    {
-        throw new IllegalStateException();
-    }
-
-    boolean validateDepsCommandsForKey(RoutableKey key, CommandsForKey evicting)
-    {
-        CommandsForKey reloaded = AccordKeyspace.loadDepsCommandsForKey(this, (PartitionKey) key);
+        CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(this, (PartitionKey) key);
         return Objects.equals(evicting, reloaded);
     }
 
-    boolean validateAllCommandsForKey(RoutableKey key, CommandsForKey evicting)
-    {
-        CommandsForKey reloaded = AccordKeyspace.loadAllCommandsForKey(this, (PartitionKey) key);
-        return Objects.equals(evicting, reloaded);
-    }
-
-    @Nullable
-    private Runnable saveCommandsForKey(CommandsForKey before, CommandsForKey after)
-    {
-        throw new IllegalStateException();
-    }
-
     @Nullable
     private Runnable saveTimestampsForKey(TimestampsForKey before, TimestampsForKey after)
     {
@@ -425,7 +369,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     }
 
     @Nullable
-    private Runnable saveCommandsForKeyUpdate(CommandsForKeyUpdate before, CommandsForKeyUpdate after)
+    private Runnable saveCommandsForKey(CommandsForKey before, CommandsForKey after)
     {
         Mutation mutation = AccordKeyspace.getCommandsForKeyMutation(id, after, nextSystemTimestampMicros());
         return null != mutation ? mutation::applyUnsafe : null;
@@ -523,15 +467,13 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     public AccordSafeCommandStore beginOperation(PreLoadContext preLoadContext,
                                                  Map<TxnId, AccordSafeCommand> commands,
                                                  NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKeys,
-                                                 NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys,
-                                                 NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
-                                                 NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys)
+                                                 NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeys)
     {
         Invariants.checkState(current == null);
         commands.values().forEach(AccordSafeState::preExecute);
-        depsCommandsForKeys.values().forEach(AccordSafeState::preExecute);
+        commandsForKeys.values().forEach(AccordSafeState::preExecute);
         timestampsForKeys.values().forEach(AccordSafeState::preExecute);
-        current = new AccordSafeCommandStore(preLoadContext, commands, timestampsForKeys, depsCommandsForKeys, allCommandsForKeys, updatesForKeys, this);
+        current = new AccordSafeCommandStore(preLoadContext, commands, timestampsForKeys, commandsForKeys, this);
         return current;
     }
 
@@ -547,7 +489,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         current = null;
     }
 
-    <O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
+    <O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
     {
         keysOrRanges = keysOrRanges.slice(slice, Routables.Slice.Minimal);
         switch (keysOrRanges.domain())
@@ -555,12 +497,8 @@ public class AccordCommandStore extends CommandStore implements CacheSize
             case Key:
             {
                 AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
-                for (CommandsForRanges.DomainInfo summary : commandsForRanges.search(keys))
-                {
-                    accumulate = map.apply(summary, summary, accumulate);
-                    if (terminate.test(accumulate))
-                        return accumulate;
-                }
+                for (CommandsSummary summary : commandsForRanges.search(keys))
+                    accumulate = map.apply(summary, accumulate);
             }
             break;
             case Range:
@@ -568,12 +506,10 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                 AbstractRanges ranges = (AbstractRanges) keysOrRanges;
                 for (Range range : ranges)
                 {
-                    CommandsForRanges.DomainInfo summary = commandsForRanges.search(range);
+                    CommandsSummary summary = commandsForRanges.search(range);
                     if (summary == null)
                         continue;
-                    accumulate = map.apply(summary, summary, accumulate);
-                    if (terminate.test(accumulate))
-                        return accumulate;
+                    accumulate = map.apply(summary, accumulate);
                 }
             }
             break;

src/java/org/apache/cassandra/service/accord/AccordCommandsForKeys.java

@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.nio.ByteBuffer;
-import java.util.function.Function;
-
-import accord.api.Key;
-import accord.impl.CommandsForKey;
-import accord.impl.CommandsForKeyGroupUpdater;
-import accord.impl.CommandsForKeyUpdater;
-import accord.primitives.RoutableKey;
-import accord.utils.Invariants;
-import accord.utils.async.AsyncChain;
-import org.apache.cassandra.concurrent.ExecutorPlus;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
-
-/**
- * Ties together the separate commands for key and commands for key update classes to make loading,
- * saving, and evicting coherent
- */
-public class AccordCommandsForKeys
-{
-    private final AccordCommandStore commandStore;
-
-    public AccordCommandsForKeys(AccordCommandStore commandStore)
-    {
-        this.commandStore = commandStore;
-    }
-
-    AccordCachingState<RoutableKey, CommandsForKey> createDepsCommandsNode(RoutableKey key, int index)
-    {
-        return new DepsCommandsCachingState(key, index);
-    }
-
-    AccordCachingState<RoutableKey, CommandsForKey> createAllCommandsNode(RoutableKey key, int index)
-    {
-        return new AllCommandsCachingState(key, index);
-    }
-
-    AccordCachingState<RoutableKey, CommandsForKeyUpdate> createUpdatesNode(RoutableKey key, int index)
-    {
-        return new UpdateCachingState(key, index);
-    }
-
-    protected static boolean hasEvictableStatus(AccordCachingState<?,?> state)
-    {
-        if (state == null)
-            return true;
-
-        switch (state.status())
-        {
-            case LOADING:
-            case SAVING:
-                return false;
-        }
-
-        return true;
-    }
-
-    boolean canEvictKey(RoutableKey key)
-    {
-        return hasEvictableStatus(commandStore.depsCommandsForKeyCache().getUnsafe(key))
-               && hasEvictableStatus(commandStore.allCommandsForKeyCache().getUnsafe(key))
-               && hasEvictableStatus(commandStore.updatesForKeyCache().getUnsafe(key));
-    }
-
-    public abstract class CommandsCachingState extends AccordCachingState<RoutableKey, CommandsForKey>
-    {
-        protected CommandsCachingState(RoutableKey key, int index)
-        {
-            super(key, index);
-        }
-
-        private CommandsForKey initializeIfNull(CommandsForKey commands)
-        {
-            if (commands != null)
-                return commands;
-            return new CommandsForKey((Key) key(), CommandsForKeySerializer.loader);
-        }
-
-        private State<RoutableKey, CommandsForKey> maybeApplyUpdates(State<RoutableKey, CommandsForKey> state)
-        {
-            if (!(state instanceof Loaded))
-                return state;
-
-            Loaded<RoutableKey, CommandsForKey> loaded = (Loaded<RoutableKey, CommandsForKey>) state;
-            CommandsForKey commands = loaded.get();
-            UpdateCachingState updates = (UpdateCachingState) commandStore.updatesForKeyCache().getUnsafe(key());
-            if (updates == null)
-                return loaded;
-
-            CommandsForKeyUpdate update = updates.getUpdateIfAvailable();
-            if (update == null)
-                return loaded;
-            CommandsForKey updated = apply(initializeIfNull(commands), update);
-            if (updated == commands)
-                return loaded;
-
-            return new Loaded<>(updated);
-        }
-
-        protected abstract CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update);
-
-        private void maybeApplyUpdates(CommandsForKeyUpdate update)
-        {
-            if (status() != Status.LOADED)
-                return;
-
-            CommandsForKey commands = get();
-            CommandsForKey updated = apply(initializeIfNull(commands), update);
-            if (commands != updated)
-                super.state(new Loaded<>(updated));
-        }
-
-        protected State<RoutableKey, CommandsForKey> state(State<RoutableKey, CommandsForKey> next)
-        {
-            Status nextStatus = next.status();
-            Invariants.checkState(nextStatus != Status.MODIFIED && nextStatus != Status.SAVING,
-                                  "CommandsForKey cannot have state %s", nextStatus);
-
-            return super.state(maybeApplyUpdates(next));
-        }
-
-        @Override
-        public boolean canEvict()
-        {
-            return canEvictKey(key());
-        }
-    }
-
-    public class DepsCommandsCachingState extends CommandsCachingState
-    {
-        public DepsCommandsCachingState(RoutableKey key, int index)
-        {
-            super(key, index);
-        }
-
-        protected CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update)
-        {
-            return update.applyToDeps(current);
-        }
-    }
-
-    public class AllCommandsCachingState extends CommandsCachingState
-    {
-        public AllCommandsCachingState(RoutableKey key, int index)
-        {
-            super(key, index);
-        }
-
-        protected CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update)
-        {
-            return update.applyToAll(current);
-        }
-    }
-
-    public class UpdateCachingState extends AccordCachingState<RoutableKey, CommandsForKeyUpdate> implements CommandsForKeyGroupUpdater.Immutable.Factory<ByteBuffer, CommandsForKeyUpdate>
-    {
-        public UpdateCachingState(RoutableKey key, int index)
-        {
-            super(key, index);
-        }
-
-        public AsyncChain<CommandsForKeyUpdate> load(ExecutorPlus executor, Function<RoutableKey, CommandsForKeyUpdate> loadFunction)
-        {
-            if (status() == Status.UNINITIALIZED)
-            {
-                CommandsForKeyUpdate initialized = CommandsForKeyUpdate.empty(key());
-                state(state().initialize(initialized));
-                return null;
-            }
-
-            return super.load(executor, loadFunction);
-        }
-
-        // update in memory cfk data with the update results
-        protected void maybeUpdateCommands(CommandsForKeyUpdate update, AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> cache)
-        {
-            CommandsCachingState commands = (CommandsCachingState) cache.getUnsafe(key());
-            if (commands == null)
-                return;
-
-            commands.maybeApplyUpdates(update);
-        }
-
-        public CommandsForKeyUpdate create(CommandsForKeyUpdater.Immutable<ByteBuffer> deps, CommandsForKeyUpdater.Immutable<ByteBuffer> all, CommandsForKeyUpdater.Immutable<ByteBuffer> common)
-        {
-            return new CommandsForKeyUpdate((PartitionKey) key(), deps, all, common);
-        }
-
-        protected State<RoutableKey, CommandsForKeyUpdate> maybeProcessModification(State<RoutableKey, CommandsForKeyUpdate> next)
-        {
-            if (!(next instanceof Modified))
-                return next;
-
-            Modified<RoutableKey, CommandsForKeyUpdate> modified = (Modified<RoutableKey, CommandsForKeyUpdate>) next;
-
-            CommandsForKeyUpdate current = modified.current;
-            maybeUpdateCommands(current, commandStore.depsCommandsForKeyCache());
-            maybeUpdateCommands(current, commandStore.allCommandsForKeyCache());
-
-            // combine in memory updates
-            current = CommandsForKeyGroupUpdater.Immutable.merge(modified.original, current, this);
-
-            return new Modified<>(null, current);
-        }
-
-        protected State<RoutableKey, CommandsForKeyUpdate> state(State<RoutableKey, CommandsForKeyUpdate> next)
-        {
-            Status nextStatus = next.status();
-            Invariants.checkState(nextStatus != Status.LOADING,
-                                  "CommandsForKeyUpdate cannot have state %s", nextStatus);
-
-            return super.state(maybeProcessModification(next));
-        }
-
-        CommandsForKeyUpdate getUpdateIfAvailable()
-        {
-            switch (status())
-            {
-                case LOADED:
-                case MODIFIED:
-                    return get();
-            }
-
-            return null;
-        }
-
-        @Override
-        public boolean canEvict()
-        {
-            return canEvictKey(key());
-        }
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java

@@ -149,7 +149,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
             }
 
             remoteSyncComplete.forEach(id -> receiveRemoteSyncComplete(id, epoch));
-            // TODO (now): disk doesn't get updated until we see our own notification, so there is an edge case where this instance notified others and fails in the middle, but Apply was already sent!  This could leave partial closed/redudant accross the cluster
+            // TODO (required): disk doesn't get updated until we see our own notification, so there is an edge case where this instance notified others and fails in the middle, but Apply was already sent!  This could leave partial closed/redudant accross the cluster
             receiveClosed(closed, epoch);
             receiveRedundant(redundant, epoch);
         }));

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -34,14 +33,11 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -49,9 +45,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.api.Key;
-import accord.impl.CommandTimeseries;
 import accord.impl.CommandsForKey;
-import accord.impl.CommandsForKeyUpdater;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.Command.WaitingOn;
@@ -153,8 +147,7 @@ import org.apache.cassandra.service.accord.serializers.ListenerSerializers;
 import org.apache.cassandra.service.accord.serializers.TopologySerializers;
 import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
 import org.apache.cassandra.transport.Dispatcher;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.MonotonicClock;
+import org.apache.cassandra.utils.Clock.Global;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.btree.BTree;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
@@ -164,6 +157,8 @@ import static accord.utils.Invariants.checkState;
 import static java.lang.String.format;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
 import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
+import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate;
+import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow;
 import static org.apache.cassandra.db.rows.BufferCell.live;
 import static org.apache.cassandra.db.rows.BufferCell.tombstone;
 import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
@@ -177,14 +172,13 @@ public class AccordKeyspace
 
     public static final String COMMANDS = "commands";
     public static final String TIMESTAMPS_FOR_KEY = "timestamps_for_key";
-    public static final String DEPS_COMMANDS_FOR_KEY = "deps_commands_for_key";
-    public static final String ALL_COMMANDS_FOR_KEY = "all_commands_for_key";
+    public static final String COMMANDS_FOR_KEY = "commands_for_key";
     public static final String TOPOLOGIES = "topologies";
     public static final String EPOCH_METADATA = "epoch_metadata";
     public static final String COMMAND_STORE_METADATA = "command_store_metadata";
 
-    public static final Set<String> TABLE_NAMES = ImmutableSet.of(COMMANDS, TIMESTAMPS_FOR_KEY, DEPS_COMMANDS_FOR_KEY,
-                                                                  ALL_COMMANDS_FOR_KEY, TOPOLOGIES, EPOCH_METADATA,
+    public static final Set<String> TABLE_NAMES = ImmutableSet.of(COMMANDS, TIMESTAMPS_FOR_KEY, COMMANDS_FOR_KEY,
+                                                                  TOPOLOGIES, EPOCH_METADATA,
                                                                   COMMAND_STORE_METADATA);
 
     private static final TupleType TIMESTAMP_TYPE = new TupleType(Lists.newArrayList(LongType.instance, LongType.instance, Int32Type.instance));
@@ -422,7 +416,6 @@ public class AccordKeyspace
               + "store_id int, "
               + "key_token blob, " // can't use "token" as this is restricted word in CQL
               + format("key %s, ", KEY_TUPLE)
-              + format("max_timestamp %s, ", TIMESTAMP_TUPLE)
               + format("last_executed_timestamp %s, ", TIMESTAMP_TUPLE)
               + "last_executed_micros bigint, "
               + format("last_write_timestamp %s, ", TIMESTAMP_TUPLE)
@@ -439,12 +432,11 @@ public class AccordKeyspace
         static final ColumnMetadata store_id = getColumn(TimestampsForKeys, "store_id");
         static final ColumnMetadata key_token = getColumn(TimestampsForKeys, "key_token");
         static final ColumnMetadata key = getColumn(TimestampsForKeys, "key");
-        public static final ColumnMetadata max_timestamp = getColumn(TimestampsForKeys, "max_timestamp");
         public static final ColumnMetadata last_executed_timestamp = getColumn(TimestampsForKeys, "last_executed_timestamp");
         public static final ColumnMetadata last_executed_micros = getColumn(TimestampsForKeys, "last_executed_micros");
         public static final ColumnMetadata last_write_timestamp = getColumn(TimestampsForKeys, "last_write_timestamp");
 
-        static final Columns columns = Columns.from(Lists.newArrayList(max_timestamp, last_executed_timestamp, last_executed_micros, last_write_timestamp));
+        static final Columns columns = Columns.from(Lists.newArrayList(last_executed_timestamp, last_executed_micros, last_write_timestamp));
 
         static ByteBuffer makePartitionKey(int storeId, Key key)
         {
@@ -475,15 +467,6 @@ public class AccordKeyspace
             return deserializeKey(partitionKeyComponents[key.position()]);
         }
 
-        @Nullable
-        public static Timestamp getMaxTimestamp(Row row)
-        {
-            Cell cell = row.getCell(max_timestamp);
-            if (cell == null)
-                return null;
-            return deserializeTimestampOrNull(cell.value(), cell.accessor(), Timestamp::fromBits);
-        }
-
         @Nullable
         public static Timestamp getLastExecutedTimestamp(Row row)
         {
@@ -510,12 +493,11 @@ public class AccordKeyspace
             return deserializeTimestampOrNull(cell.value(), cell.accessor(), Timestamp::fromBits);
         }
 
-        public static Row truncateTimestampsForKeyRow(long nowInSec, Row row, Cell lastExecuteMicrosCell, Cell lastExecuteCell, Cell lastWriteCell, Cell maxTimestampCell)
+        public static Row truncateTimestampsForKeyRow(long nowInSec, Row row, Cell lastExecuteMicrosCell, Cell lastExecuteCell, Cell lastWriteCell)
         {
             checkArgument(lastExecuteMicrosCell == null || lastExecuteMicrosCell.column() == last_executed_micros);
             checkArgument(lastExecuteCell == null || lastExecuteCell.column() == last_executed_timestamp);
             checkArgument(lastWriteCell == null || lastWriteCell.column() == last_write_timestamp);
-            checkArgument(maxTimestampCell == null || maxTimestampCell.column() == max_timestamp);
 
             long timestamp = row.primaryKeyLivenessInfo().timestamp();
 
@@ -526,8 +508,6 @@ public class AccordKeyspace
                 colCount++;
             if (lastWriteCell != null)
                 colCount++;
-            if (maxTimestampCell != null)
-                colCount++;
 
             checkState(columns.size() >= colCount, "CommandsForKeyColumns.static_columns_metadata should include all the columns");
             Object[] newLeaf = BTree.unsafeAllocateNonEmptyLeaf(colCount);
@@ -538,17 +518,14 @@ public class AccordKeyspace
             if (lastExecuteCell != null)
                 newLeaf[colIndex++] = lastExecuteCell;
             if (lastWriteCell != null)
-                newLeaf[colIndex++] = lastWriteCell;
-            if (maxTimestampCell != null)
-                newLeaf[colIndex++] = maxTimestampCell;
+                newLeaf[colIndex] = lastWriteCell;
 
             return BTreeRow.create(row.clustering(), LivenessInfo.create(timestamp, nowInSec),
                                    Deletion.LIVE, newLeaf);
         }
     }
 
-    private static final TableMetadata DepsCommandsForKeys = commandsForKeysTable(DEPS_COMMANDS_FOR_KEY);
-    private static final TableMetadata AllCommandsForKeys = commandsForKeysTable(ALL_COMMANDS_FOR_KEY);
+    private static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
 
     private static TableMetadata commandsForKeysTable(String tableName)
     {
@@ -558,9 +535,8 @@ public class AccordKeyspace
               + "store_id int, "
               + "key_token blob, " // can't use "token" as this is restricted word in CQL
               + format("key %s, ", KEY_TUPLE)
-              + format("timestamp %s, ", TIMESTAMP_TUPLE)
               + "data blob, "
-              + "PRIMARY KEY((store_id, key_token, key), timestamp)"
+              + "PRIMARY KEY((store_id, key_token, key))"
               + ')')
         .partitioner(FOR_KEYS_LOCAL_PARTITIONER)
         .build();
@@ -575,8 +551,6 @@ public class AccordKeyspace
         final ColumnMetadata store_id;
         final ColumnMetadata key_token;
         final ColumnMetadata key;
-        final ColumnMetadata timestamp;
-
         final ColumnMetadata data;
 
         final RegularAndStaticColumns columns;
@@ -590,7 +564,6 @@ public class AccordKeyspace
             this.store_id = getColumn(table, "store_id");
             this.key_token = getColumn(table, "key_token");
             this.key = getColumn(table, "key");
-            this.timestamp = getColumn(table, "timestamp");
             this.data = getColumn(table, "data");
             this.columns = new RegularAndStaticColumns(Columns.NONE, Columns.from(Lists.newArrayList(data)));
         }
@@ -610,15 +583,37 @@ public class AccordKeyspace
             return deserializeKey(partitionKeyComponents[key.position()]);
         }
 
-        @Nullable
-        public Timestamp getTimestamp(Row row)
+        public CommandsForKey getCommandsForKey(PartitionKey key, Row row)
         {
-            return deserializeTimestampOrNull(row.clustering().bufferAt(timestamp.position()), Timestamp::fromBits);
+            Cell<?> cell = row.getCell(data);
+            if (cell == null)
+                return null;
+
+            return CommandsForKeySerializer.fromBytes(key, cell.buffer());
+        }
+
+        // TODO (expected): garbage-free filtering, reusing encoding
+        public Row withoutRedundantCommands(PartitionKey key, Row row, TxnId redundantBefore)
+        {
+            Invariants.checkState(row.columnCount() == 1);
+            Cell<?> cell = row.getCell(data);
+            if (cell == null)
+                return row;
+
+            CommandsForKey current = CommandsForKeySerializer.fromBytes(key, cell.buffer());
+            CommandsForKey updated = current.withoutRedundant(redundantBefore);
+            if (current == updated)
+                return row;
+
+            if (updated.size() == 0)
+                return null;
+
+            ByteBuffer buffer = CommandsForKeySerializer.toBytesWithoutKey(updated);
+            return BTreeRow.singleCellRow(Clustering.EMPTY, BufferCell.live(data, cell.timestamp(), buffer));
         }
     }
 
-    public static final CommandsForKeyAccessor DepsCommandsForKeysAccessor = new CommandsForKeyAccessor(DepsCommandsForKeys);
-    public static final CommandsForKeyAccessor AllCommandsForKeysAccessor = new CommandsForKeyAccessor(AllCommandsForKeys);
+    public static final CommandsForKeyAccessor CommandsForKeysAccessor = new CommandsForKeyAccessor(CommandsForKeys);
 
     private static final TableMetadata Topologies =
         parse(TOPOLOGIES,
@@ -677,7 +672,7 @@ public class AccordKeyspace
 
     public static Tables tables()
     {
-        return Tables.of(Commands, TimestampsForKeys, DepsCommandsForKeys, AllCommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata);
+        return Tables.of(Commands, TimestampsForKeys, CommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata);
     }
 
     private static <T> ByteBuffer serialize(T obj, LocalVersionedSerializer<T> serializer) throws IOException
@@ -838,7 +833,7 @@ public class AccordKeyspace
             ByteBuffer key = CommandsColumns.keyComparator.make(storeId,
                                                                 command.txnId().domain().ordinal(),
                                                                 serializeTimestamp(command.txnId())).serializeAsPartitionKey();
-            PartitionUpdate update = PartitionUpdate.singleRowUpdate(Commands, key, row);
+            PartitionUpdate update = singleRowUpdate(Commands, key, row);
             return new Mutation(update);
         }
         catch (IOException e)
@@ -972,7 +967,7 @@ public class AccordKeyspace
 
     private static abstract class TableWalk implements Runnable, DebuggableTask
     {
-        private final long creationTimeNanos = Clock.Global.nanoTime();
+        private final long creationTimeNanos = Global.nanoTime();
         private final Executor executor;
         private final Observable<UntypedResultSet.Row> callback;
         private long startTimeNanos = -1;
@@ -998,7 +993,7 @@ public class AccordKeyspace
             try
             {
                 if (startTimeNanos == -1)
-                    startTimeNanos = Clock.Global.nanoTime();
+                    startTimeNanos = Global.nanoTime();
                 numQueries++;
                 UntypedResultSet result = query(lastSeen);
                 if (result.isEmpty())
@@ -1329,7 +1324,6 @@ public class AccordKeyspace
             int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
             LivenessInfo livenessInfo = LivenessInfo.create(timestampMicros, nowInSeconds);
             builder.addPrimaryKeyLivenessInfo(livenessInfo);
-            addCellIfModified(TimestampsForKeyColumns.max_timestamp, TimestampsForKey::max, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
             addCellIfModified(TimestampsForKeyColumns.last_executed_timestamp, TimestampsForKey::lastExecutedTimestamp, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
             addCellIfModified(TimestampsForKeyColumns.last_executed_micros, TimestampsForKey::rawLastExecutedHlc, accessor::valueOf, builder, timestampMicros, nowInSeconds, original, current);
             addCellIfModified(TimestampsForKeyColumns.last_write_timestamp, TimestampsForKey::lastWriteTimestamp, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
@@ -1339,7 +1333,7 @@ public class AccordKeyspace
                 return null;
 
             ByteBuffer key = TimestampsForKeyColumns.makePartitionKey(storeId, current.key());
-            PartitionUpdate update = PartitionUpdate.singleRowUpdate(TimestampsForKeys, key, row);
+            PartitionUpdate update = singleRowUpdate(TimestampsForKeys, key, row);
             return new Mutation(update);
         }
         catch (IOException e)
@@ -1374,7 +1368,6 @@ public class AccordKeyspace
 
     public static TimestampsForKey unsafeLoadTimestampsForKey(AccordCommandStore commandStore, PartitionKey key)
     {
-
         UntypedResultSet rows = loadTimestampsForKeyRow(commandStore, key);
 
         if (rows.isEmpty())
@@ -1385,60 +1378,11 @@ public class AccordKeyspace
         UntypedResultSet.Row row = rows.one();
         checkState(deserializeKey(row).equals(key));
 
-        Timestamp max = deserializeTimestampOrDefault(row, "max_timestamp", Timestamp::fromBits, Timestamp.NONE);
         Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(row, "last_executed_timestamp", Timestamp::fromBits, Timestamp.NONE);
         long lastExecutedMicros = row.has("last_executed_micros") ? row.getLong("last_executed_micros") : 0;
         Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(row, "last_write_timestamp", Timestamp::fromBits, Timestamp.NONE);
 
-        return TimestampsForKey.SerializerSupport.create(key, max, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp);
-    }
-
-    private static <T extends Timestamp> void addSeriesMutations(CommandsForKeyAccessor accessor,
-                                                                 CommandTimeseries.Update<T, ByteBuffer> update,
-                                                                 PartitionUpdate.Builder partitionBuilder,
-                                                                 Row.Builder rowBuilder,
-                                                                 LivenessInfo livenessInfo,
-                                                                 long timestampMicros,
-                                                                 Row.Deletion deletion,
-                                                                 Predicate<T> predicate)
-    {
-        if (update.isEmpty())
-            return;
-
-        update.forEachWrite((timestamp, bytes) -> {
-            if (!predicate.test(timestamp))
-                return;
-            rowBuilder.newRow(Clustering.make(serializeTimestamp(timestamp)));
-            rowBuilder.addCell(live(accessor.data, timestampMicros, bytes));
-            rowBuilder.addPrimaryKeyLivenessInfo(livenessInfo);
-            partitionBuilder.add(rowBuilder.build());
-        });
-        update.forEachDelete(timestamp -> {
-            if (!predicate.test(timestamp))
-                return;
-            rowBuilder.newRow(Clustering.make(serializeTimestamp(timestamp)));
-            rowBuilder.addRowDeletion(deletion);
-            partitionBuilder.add(rowBuilder.build());
-        });
-    }
-
-    private static <T extends Timestamp> void addSeriesMutations(CommandsForKeyAccessor accessor,
-                                                                 CommandTimeseries.Update<T, ByteBuffer> common,
-                                                                 CommandTimeseries.Update<T, ByteBuffer> update,
-                                                                 PartitionUpdate.Builder partitionBuilder,
-                                                                 Row.Builder rowBuilder,
-                                                                 LivenessInfo livenessInfo,
-                                                                 int nowInSeconds)
-    {
-
-        long timestampMicros = livenessInfo.timestamp();
-
-        Row.Deletion deletion = common.numDeletes() + update.numDeletes() > 0 ?
-                                Row.Deletion.regular(DeletionTime.build(timestampMicros, nowInSeconds)) :
-                                null;
-
-        addSeriesMutations(accessor, common, partitionBuilder, rowBuilder, livenessInfo, timestampMicros, deletion, ts -> !update.contains(ts));
-        addSeriesMutations(accessor, update, partitionBuilder, rowBuilder, livenessInfo, timestampMicros, deletion, ts -> true);
+        return TimestampsForKey.SerializerSupport.create(key, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp);
     }
 
     private static DecoratedKey makeKey(CommandsForKeyAccessor accessor, int storeId, PartitionKey key)
@@ -1450,54 +1394,17 @@ public class AccordKeyspace
         return accessor.table.partitioner.decorateKey(pk);
     }
 
-    private static PartitionUpdate getCommandsForKeyPartitionUpdate(CommandsForKeyAccessor accessor, int storeId, PartitionKey key, CommandsForKeyUpdater<ByteBuffer> common, CommandsForKeyUpdater<ByteBuffer> update, long timestampMicros)
+    private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, PartitionKey key, CommandsForKey commandsForKey, long timestampMicros)
     {
-
-        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
-        LivenessInfo livenessInfo = LivenessInfo.create(timestampMicros, nowInSeconds);
-
-        int expectedRows = common.totalChanges() + update.totalChanges();
-
-        PartitionUpdate.Builder partitionBuilder = new PartitionUpdate.Builder(accessor.table,
-                                                                               makeKey(accessor, storeId, key),
-                                                                               accessor.columns,
-                                                                               expectedRows);
-
-        Row.Builder rowBuilder = BTreeRow.unsortedBuilder();
-
-        addSeriesMutations(accessor, common.commands(), update.commands(), partitionBuilder, rowBuilder, livenessInfo, nowInSeconds);
-
-        PartitionUpdate partitionUpdate = partitionBuilder.build();
-        if (partitionUpdate.isEmpty())
-            return null;
-        return partitionUpdate;
+        ByteBuffer bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
+        return singleRowUpdate(CommandsForKeysAccessor.table,
+                               makeKey(CommandsForKeysAccessor, storeId, key),
+                               singleCellRow(Clustering.EMPTY, BufferCell.live(CommandsForKeysAccessor.data, timestampMicros, bytes)));
     }
 
-    public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKeyUpdate update, long timestampMicros)
+    public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKey update, long timestampMicros)
     {
-        PartitionUpdate depsUpdate = getCommandsForKeyPartitionUpdate(DepsCommandsForKeysAccessor,
-                                                                      storeId,
-                                                                      update.key(),
-                                                                      update.common(),
-                                                                      update.deps(),
-                                                                      timestampMicros);
-        PartitionUpdate allUpdate = getCommandsForKeyPartitionUpdate(AllCommandsForKeysAccessor,
-                                                                     storeId,
-                                                                     update.key(),
-                                                                     update.common(),
-                                                                     update.all(),
-                                                                     timestampMicros);
-
-        if (depsUpdate == null && allUpdate == null)
-            return null;
-        if (depsUpdate == null)
-            return new Mutation(allUpdate);
-        else if (allUpdate == null)
-            return new Mutation(depsUpdate);
-
-        return new Mutation(ACCORD_KEYSPACE_NAME, depsUpdate.partitionKey(),
-                            ImmutableMap.of(depsUpdate.metadata().id, depsUpdate, allUpdate.metadata().id, allUpdate),
-                            MonotonicClock.Global.approxTime.now(), false);
+        return new Mutation(getCommandsForKeyPartitionUpdate(storeId, (PartitionKey) update.key(), update, timestampMicros));
     }
 
     private static <T> ByteBuffer cellValue(Cell<T> cell)
@@ -1527,50 +1434,31 @@ public class AccordKeyspace
                                                  FULL_PARTITION);
     }
 
-    public static SinglePartitionReadCommand getDepsCommandsForKeyRead(int storeId, PartitionKey key, int nowInSeconds)
+    public static SinglePartitionReadCommand getCommandsForKeyRead(int storeId, PartitionKey key, int nowInSeconds)
     {
-        return getCommandsForKeyRead(DepsCommandsForKeysAccessor, storeId, key, nowInSeconds);
-    }
-
-    public static SinglePartitionReadCommand getAllCommandsForKeyRead(int storeId, PartitionKey key, int nowInSeconds)
-    {
-        return getCommandsForKeyRead(AllCommandsForKeysAccessor, storeId, key, nowInSeconds);
+        return getCommandsForKeyRead(CommandsForKeysAccessor, storeId, key, nowInSeconds);
     }
 
     static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, AccordCommandStore commandStore, PartitionKey key)
     {
-        long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
+        long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
         int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
 
         SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStore.id(), key, nowInSeconds);
 
-
-        ImmutableSortedMap.Builder<Timestamp, ByteBuffer> commands = new ImmutableSortedMap.Builder<>(Comparator.naturalOrder());
-
         try (ReadExecutionController controller = command.executionController();
              FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
         {
             if (!partitions.hasNext())
-            {
                 return null;
-            }
 
             try (RowIterator partition = partitions.next())
             {
-                while (partition.hasNext())
-                {
-                    Row row = partition.next();
-                    Clustering<?> clustering = row.clustering();
-                    Timestamp timestamp = deserializeTimestampOrNull(clusteringValue(clustering, 0), Timestamp::fromBits);
-                    ByteBuffer data = cellValue(row, accessor.data);
-                    if (data == null)
-                        continue;
-                    commands.put(timestamp, data);
-                }
+                Invariants.checkState(partition.hasNext());
+                Row row = partition.next();
+                ByteBuffer data = cellValue(row, accessor.data);
+                return CommandsForKeySerializer.fromBytes(key, data);
             }
-            checkState(!partitions.hasNext());
-
-            return CommandsForKey.SerializerSupport.create(key, CommandsForKeySerializer.loader, commands.build());
         }
         catch (Throwable t)
         {
@@ -1579,26 +1467,15 @@ public class AccordKeyspace
         }
     }
 
-    public static CommandsForKey unsafeLoadDepsCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
+    public static CommandsForKey unsafeLoadCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
     {
-        return unsafeLoadCommandsForKey(DepsCommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
     }
 
-    public static CommandsForKey unsafeLoadAllCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
-    {
-        return unsafeLoadCommandsForKey(AllCommandsForKeysAccessor, commandStore, key);
-    }
-
-    public static CommandsForKey loadDepsCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
+    public static CommandsForKey loadCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
     {
         commandStore.checkNotInStoreThread();
-        return unsafeLoadCommandsForKey(DepsCommandsForKeysAccessor, commandStore, key);
-    }
-
-    public static CommandsForKey loadAllCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
-    {
-        commandStore.checkNotInStoreThread();
-        return unsafeLoadCommandsForKey(AllCommandsForKeysAccessor, commandStore, key);
+        return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStore, key);
     }
 
     public static class EpochDiskState
@@ -1860,7 +1737,7 @@ public class AccordKeyspace
         ModificationStatement statement = (ModificationStatement) QueryProcessor.parseStatement(cql).prepare(ClientState.forInternalCalls());
         QueryOptions options = QueryOptions.forInternalCalls(Arrays.asList(values));
 
-        long tsMicros = TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
+        long tsMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
 
         while (true)
         {

src/java/org/apache/cassandra/service/accord/AccordMessageSink.java

@@ -124,6 +124,8 @@ public class AccordMessageSink implements MessageSink
             builder.put(MessageType.ACCEPT_INVALIDATE_REQ,                    Verb.ACCORD_ACCEPT_INVALIDATE_REQ);
             builder.put(MessageType.GET_DEPS_REQ,                             Verb.ACCORD_GET_DEPS_REQ);
             builder.put(MessageType.GET_DEPS_RSP,                             Verb.ACCORD_GET_DEPS_RSP);
+            builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_REQ,              Verb.ACCORD_GET_EPHMRL_READ_DEPS_REQ);
+            builder.put(MessageType.GET_EPHEMERAL_READ_DEPS_RSP,              Verb.ACCORD_GET_EPHMRL_READ_DEPS_RSP);
             builder.put(MessageType.COMMIT_SLOW_PATH_REQ,                     Verb.ACCORD_COMMIT_REQ);
             builder.put(MessageType.COMMIT_MAXIMAL_REQ,                       Verb.ACCORD_COMMIT_REQ);
             builder.put(MessageType.STABLE_FAST_PATH_REQ,                     Verb.ACCORD_COMMIT_REQ);
@@ -134,6 +136,7 @@ public class AccordMessageSink implements MessageSink
             builder.put(MessageType.APPLY_MAXIMAL_REQ,                        Verb.ACCORD_APPLY_REQ);
             builder.put(MessageType.APPLY_RSP,                                Verb.ACCORD_APPLY_RSP);
             builder.put(MessageType.READ_REQ,                                 Verb.ACCORD_READ_REQ);
+            builder.put(MessageType.READ_EPHEMERAL_REQ,                       Verb.ACCORD_READ_REQ);
             builder.put(MessageType.READ_RSP,                                 Verb.ACCORD_READ_RSP);
             builder.put(MessageType.BEGIN_RECOVER_REQ,                        Verb.ACCORD_BEGIN_RECOVER_REQ);
             builder.put(MessageType.BEGIN_RECOVER_RSP,                        Verb.ACCORD_BEGIN_RECOVER_RSP);
@@ -142,7 +145,7 @@ public class AccordMessageSink implements MessageSink
             builder.put(MessageType.WAIT_ON_COMMIT_REQ,                       Verb.ACCORD_WAIT_ON_COMMIT_REQ);
             builder.put(MessageType.WAIT_ON_COMMIT_RSP,                       Verb.ACCORD_WAIT_ON_COMMIT_RSP);
             builder.put(MessageType.WAIT_UNTIL_APPLIED_REQ,                   Verb.ACCORD_WAIT_UNTIL_APPLIED_REQ);
-            builder.put(MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ,        Verb.ACCORD_APPLY_AND_WAIT_UNTIL_APPLIED_REQ);
+            builder.put(MessageType.APPLY_THEN_WAIT_UNTIL_APPLIED_REQ,        Verb.ACCORD_APPLY_AND_WAIT_REQ);
             builder.put(MessageType.INFORM_OF_TXN_REQ,                        Verb.ACCORD_INFORM_OF_TXN_REQ);
             builder.put(MessageType.INFORM_DURABLE_REQ,                       Verb.ACCORD_INFORM_DURABLE_REQ);
             builder.put(MessageType.INFORM_HOME_DURABLE_REQ,                  Verb.ACCORD_INFORM_HOME_DURABLE_REQ);

src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java

@@ -18,17 +18,14 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.nio.ByteBuffer;
-import java.util.Map;
 import java.util.UUID;
 import java.util.function.ToLongFunction;
 
-import com.google.common.collect.ImmutableSortedMap;
-
 import accord.api.Key;
 import accord.api.Result;
 import accord.api.RoutingKey;
 import accord.impl.CommandsForKey;
+import accord.impl.CommandsForKey.Info;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.Command.WaitingOn;
@@ -69,7 +66,6 @@ import org.apache.cassandra.service.accord.txn.TxnQuery;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.service.accord.txn.TxnWrite;
-import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 
 import static org.apache.cassandra.utils.ObjectSizes.measure;
@@ -259,16 +255,10 @@ public class AccordObjectSizes
     }
 
     private static final long EMPTY_COMMAND_LISTENER = measure(new Command.ProxyListener(null));
-    private static final long EMPTY_CFK_LISTENER = measure(new CommandsForKey.Listener((Key) null));
-    private static final long EMPTY_CFR_LISTENER = measure(new CommandsForRanges.Listener(null));
     public static long listener(Command.DurableAndIdempotentListener listener)
     {
         if (listener instanceof Command.ProxyListener)
             return EMPTY_COMMAND_LISTENER + timestamp(((Command.ProxyListener) listener).txnId());
-        if (listener instanceof CommandsForKey.Listener)
-            return EMPTY_CFK_LISTENER + key(((CommandsForKey.Listener) listener).key());
-        if (listener instanceof CommandsForRanges.Listener)
-            return EMPTY_CFR_LISTENER + timestamp(((CommandsForRanges.Listener) listener).txnId);
         throw new IllegalArgumentException("Unhandled listener type: " + listener.getClass());
     }
 
@@ -362,34 +352,34 @@ public class AccordObjectSizes
         return size;
     }
 
-    private static long cfkSeriesSize(ImmutableSortedMap<Timestamp, ByteBuffer> series)
-    {
-        long size = 0;
-        for (Map.Entry<Timestamp, ByteBuffer> entry : series.entrySet())
-        {
-            size += timestamp(entry.getKey());
-            size += ByteBufferUtil.estimatedSizeOnHeap(entry.getValue());
-        }
-        return size;
-    }
-
-    private static long EMPTY_TFK_SIZE = measure(TimestampsForKey.SerializerSupport.create(null, null, null, 0, null));
+    private static long EMPTY_TFK_SIZE = measure(TimestampsForKey.SerializerSupport.create(null, null, 0, null));
 
     public static long timestampsForKey(TimestampsForKey timestamps)
     {
         long size = EMPTY_TFK_SIZE;
-        size += timestamp(timestamps.max());
         size += timestamp(timestamps.lastExecutedTimestamp());
         size += timestamp(timestamps.lastWriteTimestamp());
         return size;
     }
 
-    private static long EMPTY_CFK_SIZE = measure(CommandsForKey.SerializerSupport.create(null, null, ImmutableSortedMap.of()));
+    private static long EMPTY_CFK_SIZE = measure(new CommandsForKey(null));
+    private static long EMPTY_INFO_SIZE = measure(CommandsForKey.Info.createMock(null, null, null));
     public static long commandsForKey(CommandsForKey cfk)
     {
         long size = EMPTY_CFK_SIZE;
         size += key(cfk.key());
-        size += cfkSeriesSize((ImmutableSortedMap<Timestamp, ByteBuffer>) cfk.commands().commands);
+        size += 2 * ObjectSizes.sizeOfReferenceArray(cfk.size());
+        size += cfk.size() * TIMESTAMP_SIZE;
+        for (int i = 0 ; i < cfk.size() ; ++i)
+        {
+            Info info = cfk.info(i);
+            if (info.getClass() == CommandsForKey.NoInfo.class)
+                continue;
+
+            size += EMPTY_INFO_SIZE;
+            if (info.missing.length > 0)
+                size += ObjectSizes.sizeOfReferenceArray(info.missing.length);
+        }
         return size;
     }
 }

src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java

@@ -18,53 +18,43 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
-import java.util.function.Predicate;
-import javax.annotation.Nullable;
+import java.util.function.BiFunction;
 
-import com.google.common.base.Predicates;
+import javax.annotation.Nullable;
 
 import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.impl.AbstractSafeCommandStore;
-import accord.impl.CommandTimeseries;
-import accord.impl.CommandTimeseries.CommandLoader;
 import accord.impl.CommandsForKey;
 import accord.impl.CommandsForKeys;
-import accord.impl.DomainCommands;
-import accord.impl.DomainTimestamps;
-import accord.impl.SafeTimestampsForKey;
+import accord.impl.CommandsSummary;
+import accord.local.Command;
 import accord.local.CommandStores.RangesForEpoch;
-import accord.local.CommonAttributes;
-import accord.local.KeyHistory;
 import accord.local.NodeTimeService;
 import accord.local.PreLoadContext;
-import accord.local.SafeCommand;
-import accord.local.Status;
 import accord.primitives.AbstractKeys;
 import accord.primitives.Deps;
-import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.RoutableKey;
 import accord.primitives.Routables;
-import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
-import accord.utils.TriFunction;
-import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 
-public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey, AccordSafeCommandsForKeyUpdate>
+import static accord.primitives.Routable.Domain.Range;
+
+public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey>
 {
     private final Map<TxnId, AccordSafeCommand> commands;
-    private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys;
-    private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys;
+    private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeys;
     private final NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKeys;
-    private final NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys;
     private final AccordCommandStore commandStore;
     private final RangesForEpoch ranges;
     CommandsForRanges.Updater rangeUpdates = null;
@@ -72,17 +62,13 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
     public AccordSafeCommandStore(PreLoadContext context,
                                   Map<TxnId, AccordSafeCommand> commands,
                                   NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey,
-                                  NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKey,
-                                  NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
-                                  NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys,
+                                  NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKey,
                                   AccordCommandStore commandStore)
     {
         super(context);
         this.commands = commands;
         this.timestampsForKeys = timestampsForKey;
-        this.depsCommandsForKeys = depsCommandsForKey;
-        this.allCommandsForKeys = allCommandsForKeys;
-        this.updatesForKeys = updatesForKeys;
+        this.commandsForKeys = commandsForKey;
         this.commandStore = commandStore;
         this.ranges = commandStore.updateRangesForEpoch();
     }
@@ -107,55 +93,22 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         return command;
     }
 
-    private NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeyMap(KeyHistory history)
+    @Override
+    protected AccordSafeCommandsForKey getCommandsForKeyInternal(RoutableKey key)
     {
-        switch (history)
-        {
-            case DEPS:
-                return depsCommandsForKeys;
-            case ALL:
-                return allCommandsForKeys;
-            default:
-                throw new IllegalArgumentException();
-        }
+        return commandsForKeys.get(key);
     }
 
     @Override
-    protected AccordSafeCommandsForKey getDepsCommandsForKeyInternal(RoutableKey key)
+    protected void addCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
     {
-        return depsCommandsForKeys.get(key);
+        commandsForKeys.put(cfk.key(), cfk);
     }
 
     @Override
-    protected void addDepsCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
+    protected AccordSafeCommandsForKey getCommandsForKeyIfLoaded(RoutableKey key)
     {
-        depsCommandsForKeys.put(cfk.key(), cfk);
-    }
-
-    @Override
-    protected AccordSafeCommandsForKey getDepsCommandsForKeyIfLoaded(RoutableKey key)
-    {
-        AccordSafeCommandsForKey cfk = commandStore.depsCommandsForKeyCache().acquireIfLoaded(key);
-        if (cfk != null) cfk.preExecute();
-        return cfk;
-    }
-
-    @Override
-    protected AccordSafeCommandsForKey getAllCommandsForKeyInternal(RoutableKey key)
-    {
-        return allCommandsForKeys.get(key);
-    }
-
-    @Override
-    protected void addAllCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
-    {
-        allCommandsForKeys.put(cfk.key(), cfk);
-    }
-
-    @Override
-    protected AccordSafeCommandsForKey getAllCommandsForKeyIfLoaded(RoutableKey key)
-    {
-        AccordSafeCommandsForKey cfk = commandStore.allCommandsForKeyCache().acquireIfLoaded(key);
+        AccordSafeCommandsForKey cfk = commandStore.commandsForKeyCache().acquireIfLoaded(key);
         if (cfk != null) cfk.preExecute();
         return cfk;
     }
@@ -180,27 +133,6 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         return cfk;
     }
 
-    protected AccordSafeCommandsForKeyUpdate getCommandsForKeyUpdateInternal(RoutableKey key)
-    {
-        return updatesForKeys.get(key);
-    }
-
-    protected AccordSafeCommandsForKeyUpdate createCommandsForKeyUpdateInternal(RoutableKey key)
-    {
-        throw new IllegalStateException("CFK updates should be initialized for operation");
-    }
-
-    protected void addCommandsForKeyUpdateInternal(AccordSafeCommandsForKeyUpdate update)
-    {
-        updatesForKeys.put(update.key(), update);
-    }
-
-    protected void applyCommandForKeyUpdates()
-    {
-        // TODO (now): should this happen as part of invalidate? Less obvious it's happening, but eliminates possibility of post update changes
-        updatesForKeys.values().forEach(AccordSafeCommandsForKeyUpdate::setUpdates);
-    }
-
     @Override
     public AccordCommandStore commandStore()
     {
@@ -237,19 +169,6 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         return commandStore().unsafeRangesForEpoch();
     }
 
-    @Override
-    public long latestEpoch()
-    {
-        return commandStore().time().epoch();
-    }
-
-    @Override
-    public Timestamp maxConflict(Seekables<?, ?> keysOrRanges, Ranges slice)
-    {
-        Timestamp maxConflict = mapReduce(keysOrRanges, slice, KeyHistory.NONE, (ts, commands, accum) -> Timestamp.max(ts.max(), accum), Timestamp.NONE, Predicates.isNull());
-        return Timestamp.nonNullOrMax(maxConflict, commandStore.commandsForRanges().maxRedundant());
-    }
-
     @Override
     public void registerHistoricalTransactions(Deps deps)
     {
@@ -257,6 +176,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         // We find a set of dependencies for a range then update CommandsFor to know about them
         Ranges allRanges = ranges.all();
         deps.keyDeps.keys().forEach(allRanges, key -> {
+            // TODO (now): batch register to minimise GC
             deps.keyDeps.forEach(key, txnId -> {
                 // TODO (desired, efficiency): this can be made more efficient by batching by epoch
                 if (ranges.coordinates(txnId).contains(key))
@@ -282,20 +202,13 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         });
     }
 
-    @Override
-    public void erase(SafeCommand safeCommand)
+    private <O> O mapReduce(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
     {
+        accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate);
+        return mapReduceForKey(keysOrRanges, slice, map, accumulate);
     }
 
-    private <O> O mapReduce(Routables<?> keysOrRanges, Ranges slice, KeyHistory keyHistory, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
-    {
-        accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate, terminate);
-        if (terminate.test(accumulate))
-            return accumulate;
-        return mapReduceForKey(keysOrRanges, slice, keyHistory, map, accumulate, terminate);
-    }
-
-    private <O> O mapReduceForKey(Routables<?> keysOrRanges, Ranges slice, KeyHistory keyHistory, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
+    private <O> O mapReduceForKey(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
     {
         switch (keysOrRanges.domain())
         {
@@ -308,11 +221,8 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
                 for (Key key : keys)
                 {
                     if (!slice.contains(key)) continue;
-                    SafeTimestampsForKey timestamps = timestampsForKey(key);
-                    CommandsForKey commands = !keyHistory.isNone() ? commandsForKey(key, keyHistory).current() : null;
-                    accumulate = map.apply(timestamps.current(), commands, accumulate);
-                    if (terminate.test((accumulate)))
-                        return accumulate;
+                    CommandsForKey commands = commandsForKey(key).current();
+                    accumulate = map.apply(commands, accumulate);
                 }
             }
             break;
@@ -327,11 +237,8 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
                 {
                     //TODO (duplicate code): this is a repeat of Key... only change is checking contains in range
                     if (!sliced.contains(key)) continue;
-                    SafeTimestampsForKey timestamps = timestampsForKey(key);
-                    CommandsForKey commands = !keyHistory.isNone() ? commandsForKey(key, keyHistory).current() : null;
-                    accumulate = map.apply(timestamps.current(), commands, accumulate);
-                    if (terminate.test(accumulate))
-                        return accumulate;
+                    CommandsForKey commands = commandsForKey(key).current();
+                    accumulate = map.apply(commands, accumulate);
                 }
             }
             break;
@@ -340,85 +247,41 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
     }
 
     @Override
-//<<<<<<< HEAD
-//    public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, Txn.Kind.Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate)   {
-//        accumulate = mapReduce(keysOrRanges, slice, (forKey, prev) -> {
-//            CommandTimeseries<?> timeseries;
-//=======
-    public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, KeyHistory keyHistory, Txn.Kind.Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate)
+    public <P1, T> T mapReduceActive(Seekables<?, ?> keysOrRanges, Ranges slice, @Nullable Timestamp withLowerTxnId, Txn.Kind.Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
     {
-        accumulate = mapReduce(keysOrRanges, slice, keyHistory, (timestamps, commands, prev) -> {
-            CommandTimeseries.TimestampType timestampType;
-            switch (testTimestamp)
-            {
-                default: throw new AssertionError();
-                case STARTED_AFTER:
-                case STARTED_BEFORE:
-                    timestampType = CommandTimeseries.TimestampType.TXN_ID;
-                    break;
-                case EXECUTES_AFTER:
-                case MAY_EXECUTE_BEFORE:
-                    timestampType = CommandTimeseries.TimestampType.EXECUTE_AT;
-            }
-            CommandTimeseries.TestTimestamp remapTestTimestamp;
-            switch (testTimestamp)
-            {
-                default: throw new AssertionError();
-                case STARTED_AFTER:
-                case EXECUTES_AFTER:
-                    remapTestTimestamp = CommandTimeseries.TestTimestamp.AFTER;
-                    break;
-                case STARTED_BEFORE:
-                case MAY_EXECUTE_BEFORE:
-                    remapTestTimestamp = CommandTimeseries.TestTimestamp.BEFORE;
-            }
-            return commands.commands().mapReduce(testKind, timestampType, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, prev, terminate);
-        }, accumulate, terminate);
-
-        return accumulate;
+        return mapReduce(keysOrRanges, slice, (summary, in) -> {
+            return summary.mapReduceActive(withLowerTxnId, testKind, map, p1, in);
+        }, accumulate);
     }
 
     @Override
-    public CommonAttributes completeRegistration(Seekables<?, ?> seekables, Ranges ranges, AccordSafeCommand liveCommand, CommonAttributes attrs)
+    public <P1, T> T mapReduceFull(Seekables<?, ?> keysOrRanges, Ranges slice, TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
     {
-        for (Seekable seekable : seekables)
-            attrs = completeRegistration(seekable, ranges, liveCommand, attrs);
-        return attrs;
+        return mapReduce(keysOrRanges, slice, (summary, in) -> {
+            return summary.mapReduceFull(testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, in);
+        }, accumulate);
     }
 
     @Override
-    public CommonAttributes completeRegistration(Seekable seekable, Ranges ranges, AccordSafeCommand liveCommand, CommonAttributes attrs)
+    protected void update(Command prev, Command updated, @Nullable Seekables<?, ?> keysOrRanges)
     {
-        switch (seekable.domain())
+        super.update(prev, updated, keysOrRanges);
+
+        if (updated.txnId().domain() == Range && CommandsForKey.needsUpdate(prev, updated))
         {
-            case Key:
+            if (keysOrRanges == null)
             {
-                Key key = seekable.asKey();
-                if (ranges.contains(key))
-                {
-                    CommandsForKeys.registerCommand(this, key, liveCommand.current());
-                    attrs = attrs.mutable().addListener(new CommandsForKey.Listener(key));
-                }
+                if (updated.known().isDefinitionKnown()) keysOrRanges = updated.partialTxn().keys();
+                else if (prev.known().isDefinitionKnown()) keysOrRanges = prev.partialTxn().keys();
+                else return;
             }
-            break;
-            case Range:
-                Range range = seekable.asRange();
-                if (!ranges.intersects(range))
-                    return attrs;
-                // TODO (api) : cleaner way to deal with this?  This is tracked at the Ranges level and not Range level
-                // but we register at the Range level...
-                if (!attrs.durableListeners().stream().anyMatch(l -> l instanceof CommandsForRanges.Listener))
-                {
-                    CommandsForRanges.Listener listener = new CommandsForRanges.Listener(liveCommand.txnId());
-                    attrs = attrs.mutable().addListener(listener);
-                    // trigger to allow it to run right away
-                    listener.onChange(this, liveCommand);
-                }
-                break;
-            default:
-                throw new UnsupportedOperationException("Unknown domain: " + seekable.domain());
+            List<TxnId> waitingOn;
+
+            if (updated.partialDeps() == null) waitingOn = Collections.emptyList();
+            // TODO (required): this is faulty: we cannot simply save the raw transaction ids, as they may be for other ranges
+            else waitingOn = updated.partialDeps().txnIds();
+            updateRanges().put(updated.txnId(), (Ranges)keysOrRanges, updated.saveStatus(), updated.executeAt(), waitingOn);
         }
-        return attrs;
     }
 
     protected CommandsForRanges.Updater updateRanges()
@@ -433,30 +296,18 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
     {
         commands.values().forEach(AccordSafeCommand::invalidate);
         timestampsForKeys.values().forEach(AccordSafeTimestampsForKey::invalidate);
-        depsCommandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
-        allCommandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
-        updatesForKeys.values().forEach(AccordSafeCommandsForKeyUpdate::invalidate);
-    }
-
-    @Override
-    public CommandLoader<?> cfkLoader(RoutableKey key)
-    {
-        return CommandsForKeySerializer.loader;
+        commandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
     }
 
     public void postExecute(Map<TxnId, AccordSafeCommand> commands,
                             Map<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey,
-                            Map<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys,
-                            Map<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
-                            Map<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys
+                            Map<RoutableKey, AccordSafeCommandsForKey> commandsForKeys
                             )
     {
         postExecute();
         commands.values().forEach(AccordSafeState::postExecute);
         timestampsForKey.values().forEach(AccordSafeState::postExecute);
-        depsCommandsForKeys.values().forEach(AccordSafeState::postExecute);
-        allCommandsForKeys.values().forEach(AccordSafeState::postExecute);
-        updatesForKeys.values().forEach(AccordSafeState::postExecute);
+        commandsForKeys.values().forEach(AccordSafeState::postExecute);
         if (rangeUpdates != null)
             rangeUpdates.apply();
     }

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java

@@ -76,7 +76,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
         // cfk initialization is legal, but doesn't need to be propagated to the cache (and would
         // cause an exception to be thrown if it were). Making an exception on the cache side could
         // throw away applied cfk updates as well, so it's special cased here
-        if (hasUpdate && original == null && current != null && current.commands().isEmpty())
+        if (hasUpdate && original == null && current != null && current.size() == 0)
             return false;
 
         return hasUpdate;
@@ -122,7 +122,8 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
     public void postExecute()
     {
         checkNotInvalidated();
-        // updates are applied directly by CommandsForKeyUpdate
+        if (current != original)
+            global.set(current);
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKeyUpdate.java

@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.nio.ByteBuffer;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import accord.api.Key;
-import accord.impl.SafeCommandsForKey;
-import accord.primitives.RoutableKey;
-import accord.utils.async.AsyncChain;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
-
-public class AccordSafeCommandsForKeyUpdate extends SafeCommandsForKey.Update<CommandsForKeyUpdate, ByteBuffer> implements AccordSafeState<RoutableKey, CommandsForKeyUpdate>
-{
-    private boolean invalidated;
-    private final AccordCachingState<RoutableKey, CommandsForKeyUpdate> global;
-    private CommandsForKeyUpdate original;
-    private CommandsForKeyUpdate current;
-
-    public AccordSafeCommandsForKeyUpdate(AccordCachingState<RoutableKey, CommandsForKeyUpdate> global)
-    {
-        super((Key) global.key(), CommandsForKeySerializer.loader);
-        this.global = global;
-        this.original = null;
-        this.current = null;
-    }
-
-    @Override
-    public void initialize()
-    {
-        set(CommandsForKeyUpdate.empty((PartitionKey) key()));
-    }
-
-    @Override
-    public AccordCachingState<RoutableKey, CommandsForKeyUpdate> global()
-    {
-        checkNotInvalidated();
-        return global;
-    }
-
-    @Override
-    public CommandsForKeyUpdate current()
-    {
-        checkNotInvalidated();
-        return current;
-    }
-
-    public AsyncChain<?> loading()
-    {
-        throw new IllegalStateException("Updates aren't loaded");
-    }
-
-    @Override
-    @VisibleForTesting
-    public void set(CommandsForKeyUpdate cfk)
-    {
-        checkNotInvalidated();
-        this.current = cfk;
-    }
-
-    public CommandsForKeyUpdate original()
-    {
-        checkNotInvalidated();
-        return original;
-    }
-
-    public CommandsForKeyUpdate setUpdates()
-    {
-        CommandsForKeyUpdate next = new CommandsForKeyUpdate((PartitionKey) key(),
-                                                             deps().toImmutable(),
-                                                             all().toImmutable(),
-                                                             common().toImmutable());
-        set(next);
-        return next;
-    }
-
-    @Override
-    public void preExecute()
-    {
-        checkNotInvalidated();
-        original = global.get();
-        current = original;
-    }
-
-    @Override
-    public void postExecute()
-    {
-        checkNotInvalidated();
-        global.set(current);
-    }
-
-    @Override
-    public void invalidate()
-    {
-        invalidated = true;
-    }
-
-    @Override
-    public boolean invalidated()
-    {
-        return invalidated;
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -30,9 +30,12 @@ import javax.annotation.concurrent.GuardedBy;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
 
 import accord.coordinate.TopologyMismatch;
+import accord.impl.CoordinateDurabilityScheduling;
 import org.apache.cassandra.cql3.statements.RequestValidations;
+import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.transformations.AddAccordTable;
 import org.slf4j.Logger;
@@ -89,9 +92,6 @@ import org.apache.cassandra.service.accord.api.AccordTopologySorter;
 import org.apache.cassandra.service.accord.api.CompositeTopologySorter;
 import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
 import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
-import org.apache.cassandra.service.accord.interop.AccordInteropApply;
-import org.apache.cassandra.service.accord.interop.AccordInteropExecution;
-import org.apache.cassandra.service.accord.interop.AccordInteropPersist;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
@@ -110,6 +110,7 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 import static accord.messages.SimpleReply.Ok;
 import static accord.utils.Invariants.checkState;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
@@ -133,6 +134,7 @@ public class AccordService implements IAccordService, Shutdownable
     private final AccordScheduler scheduler;
     private final AccordDataStore dataStore;
     private final AccordJournal journal;
+    private final CoordinateDurabilityScheduling durabilityScheduling;
     private final AccordVerbHandler<? extends Request> verbHandler;
     private final LocalConfig configuration;
     @GuardedBy("this")
@@ -307,11 +309,10 @@ public class AccordService implements IAccordService, Shutdownable
                                                             new AccordTopologySorter.Supplier(configService, DatabaseDescriptor.getNodeProximity())),
                              SimpleProgressLog::new,
                              AccordCommandStores.factory(journal),
-                             new AccordInteropExecution.Factory(agent, configService),
-                             AccordInteropPersist.FACTORY,
-                             AccordInteropApply.FACTORY,
+                             new AccordInteropFactory(agent, configService),
                              configuration);
         this.nodeShutdown = toShutdownable(node);
+        this.durabilityScheduling = new CoordinateDurabilityScheduling(node);
         this.verbHandler = new AccordVerbHandler<>(node, configService, journal);
     }
 
@@ -325,6 +326,11 @@ public class AccordService implements IAccordService, Shutdownable
         ClusterMetadataService.instance().log().addListener(configService);
         fastPathCoordinator.start();
         ClusterMetadataService.instance().log().addListener(fastPathCoordinator);
+        durabilityScheduling.setGlobalCycleTime(Ints.checkedCast(DatabaseDescriptor.getAccordGlobalDurabilityCycle(SECONDS)), SECONDS);
+        durabilityScheduling.setShardCycleTime(Ints.checkedCast(DatabaseDescriptor.getAccordShardDurabilityCycle(SECONDS)), SECONDS);
+        durabilityScheduling.setTxnIdLag(Ints.checkedCast(DatabaseDescriptor.getAccordScheduleDurabilityTxnIdLag(SECONDS)), TimeUnit.SECONDS);
+        durabilityScheduling.setFrequency(Ints.checkedCast(DatabaseDescriptor.getAccordScheduleDurabilityFrequency(SECONDS)), SECONDS);
+        durabilityScheduling.start();
         state = State.STARTED;
     }
 

src/java/org/apache/cassandra/service/accord/AccordStateCache.java

@@ -143,8 +143,6 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
             AccordCachingState<?, ?> node = iter.next();
             checkState(node.references == 0);
 
-            if (!node.canEvict())
-                continue;
             /*
              * TODO (expected, efficiency):
              *    can this be reworked so we're not skipping unevictable nodes everytime we try to evict?

src/java/org/apache/cassandra/service/accord/CommandsForKeyUpdate.java

@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-import accord.impl.CommandTimeseries;
-import accord.impl.CommandsForKey;
-import accord.impl.CommandsForKeyGroupUpdater;
-import accord.impl.CommandsForKeyUpdater;
-import accord.primitives.RoutableKey;
-import accord.primitives.Timestamp;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-
-import static org.apache.cassandra.utils.ObjectSizes.measure;
-
-public class CommandsForKeyUpdate extends CommandsForKeyGroupUpdater.Immutable<ByteBuffer> implements CommandsForKey.Update
-{
-    private static final CommandsForKeyUpdate EMPTY = new CommandsForKeyUpdate(null, null, null, null);
-
-    static long EMPTY_SIZE = measure(EMPTY);
-
-    private static long EMPTY_TIMESERIES_UPDATE_SIZE = measure(new CommandTimeseries.ImmutableUpdate(ImmutableMap.of(), ImmutableSet.of()));
-
-    private static <T extends Timestamp> long immutableTimeseriesUpdate(CommandTimeseries.ImmutableUpdate<T, ByteBuffer> update)
-    {
-        long size = EMPTY_TIMESERIES_UPDATE_SIZE;
-        for (Map.Entry<T, ByteBuffer> write : update.writes.entrySet())
-        {
-            size += AccordObjectSizes.timestamp(write.getKey());
-            size += ByteBufferAccessor.instance.size(write.getValue());
-        }
-
-        for (T delete : update.deletes)
-            size += AccordObjectSizes.timestamp(delete);
-
-        return size;
-    }
-
-    private static long EMPTY_UPDATER_SIZE = measure(new CommandsForKeyUpdater.Immutable<>(null));
-
-    private static long updaterSize(CommandsForKeyUpdater.Immutable<ByteBuffer> updater)
-    {
-        long size = EMPTY_UPDATER_SIZE;
-        size += immutableTimeseriesUpdate(updater.commands());
-        return size;
-    }
-
-    private final PartitionKey key;
-
-    public CommandsForKeyUpdate(PartitionKey key, CommandsForKeyUpdater.Immutable<ByteBuffer> deps, CommandsForKeyUpdater.Immutable<ByteBuffer> all, CommandsForKeyUpdater.Immutable<ByteBuffer> common)
-    {
-        super(deps, all, common);
-        this.key = key;
-    }
-
-    public static CommandsForKeyUpdate empty(RoutableKey key)
-    {
-        return new CommandsForKeyUpdate((PartitionKey) key,
-                                        CommandsForKeyUpdater.Immutable.empty(),
-                                        CommandsForKeyUpdater.Immutable.empty(),
-                                        CommandsForKeyUpdater.Immutable.empty());
-
-    }
-
-    public PartitionKey key()
-    {
-        return key;
-    }
-
-    public long estimatedSizeOnHeap()
-    {
-        long size = EMPTY_SIZE;
-        size += AccordObjectSizes.key(key.asKey());
-        size += updaterSize(deps());
-        size += updaterSize(all());
-        size += updaterSize(common());
-        return size;
-    }
-}

src/java/org/apache/cassandra/service/accord/CommandsForRanges.java

@@ -31,6 +31,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.function.BiFunction;
 import java.util.function.Function;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -40,23 +41,17 @@ import com.google.common.collect.ImmutableSortedMap;
 
 import accord.api.Key;
 import accord.api.RoutingKey;
-import accord.impl.CommandTimeseries;
-import accord.impl.DomainCommands;
-import accord.impl.DomainTimestamps;
+import accord.impl.CommandsForKey;
+import accord.impl.CommandsSummary;
 import accord.local.Command;
-import accord.local.PreLoadContext;
-import accord.local.SafeCommand;
-import accord.local.SafeCommandStore;
 import accord.local.SaveStatus;
 import accord.primitives.AbstractKeys;
-import accord.primitives.PartialDeps;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
-import accord.primitives.Routable;
 import accord.primitives.RoutableKey;
 import accord.primitives.Seekable;
-import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
+import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import org.apache.cassandra.schema.TableId;
@@ -66,10 +61,17 @@ import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.utils.Interval;
 import org.apache.cassandra.utils.IntervalTree;
 
+import static accord.local.SafeCommandStore.*;
+import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
+import static accord.local.SafeCommandStore.TestDep.WITH;
+import static accord.local.SafeCommandStore.TestStartedAt.ANY;
+import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
+import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
+import static accord.local.Status.Stable;
+import static accord.local.Status.Truncated;
+
 public class CommandsForRanges
 {
-    public interface DomainInfo extends DomainCommands, DomainTimestamps {}
-
     public enum TxnType
     {
         UNKNOWN, LOCAL, REMOTE;
@@ -145,42 +147,6 @@ public class CommandsForRanges
         }
     }
 
-    private enum RangeCommandSummaryLoader implements CommandTimeseries.CommandLoader<RangeCommandSummary>
-    {
-        INSTANCE;
-
-        @Override
-        public RangeCommandSummary saveForCFK(Command command)
-        {
-            //TODO split write from read?
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public TxnId txnId(RangeCommandSummary data)
-        {
-            return data.txnId;
-        }
-
-        @Override
-        public Timestamp executeAt(RangeCommandSummary data)
-        {
-            return data.executeAt;
-        }
-
-        @Override
-        public SaveStatus saveStatus(RangeCommandSummary data)
-        {
-            return data.status;
-        }
-
-        @Override
-        public List<TxnId> depsIds(RangeCommandSummary data)
-        {
-            return data.deps;
-        }
-    }
-
     public static abstract class AbstractBuilder<T extends AbstractBuilder<T>>
     {
         protected final Set<TxnId> localTxns = new HashSet<>();
@@ -314,50 +280,6 @@ public class CommandsForRanges
         }
     }
 
-    public static class Listener implements Command.DurableAndIdempotentListener
-    {
-        public final TxnId txnId;
-        private transient SaveStatus saveStatus;
-
-        public Listener(TxnId txnId)
-        {
-            this.txnId = txnId;
-        }
-
-        @Override
-        public void onChange(SafeCommandStore safeStore, SafeCommand safeCommand)
-        {
-            Command current = safeCommand.current();
-            if (current.saveStatus() == saveStatus)
-                return;
-            saveStatus = current.saveStatus();
-            PartialDeps deps = current.partialDeps();
-            if (deps == null)
-                return;
-            Seekables<?, ?> keysOrRanges = current.partialTxn().keys();
-            Invariants.checkArgument(keysOrRanges.domain() == Routable.Domain.Range, "Expected txn %s to be a Range txn, but was a %s", txnId, keysOrRanges.domain());
-
-            List<TxnId> dependsOn = deps.txnIds();
-            ((AccordSafeCommandStore) safeStore).updateRanges()
-                                                .put(txnId, (Ranges) keysOrRanges, current.saveStatus(), current.executeAt(), dependsOn);
-        }
-
-        @Override
-        public PreLoadContext listenerPreLoadContext(TxnId caller)
-        {
-            return caller.equals(txnId) ? PreLoadContext.contextFor(txnId) : PreLoadContext.contextFor(txnId, Collections.singletonList(caller));
-        }
-
-        @Override
-        public String toString()
-        {
-            return "Listener{" +
-                   "txnId=" + txnId +
-                   ", saveStatus=" + saveStatus +
-                   '}';
-        }
-    }
-
     private ImmutableSet<TxnId> localCommands;
     private ImmutableSortedMap<TxnId, RangeCommandSummary> commandsToRanges;
     private IntervalTree<RoutableKey, RangeCommandSummary, Interval<RoutableKey, RangeCommandSummary>> rangesToCommands;
@@ -401,24 +323,29 @@ public class CommandsForRanges
         return maxRedundant;
     }
 
+    public static boolean needsUpdate(Command prev, Command updated)
+    {
+        return CommandsForKey.needsUpdate(prev, updated);
+    }
+
     public boolean containsLocally(TxnId txnId)
     {
         return localCommands.contains(txnId);
     }
 
-    public Iterable<DomainInfo> search(AbstractKeys<Key> keys)
+    public Iterable<CommandsSummary> search(AbstractKeys<Key> keys)
     {
         // group by the table, as ranges are based off TokenKey, which is scoped to a range
         Map<TableId, List<Key>> groupByTable = new TreeMap<>();
         for (Key key : keys)
             groupByTable.computeIfAbsent(((PartitionKey) key).table(), ignore -> new ArrayList<>()).add(key);
-        return () -> new AbstractIterator<DomainInfo>()
+        return () -> new AbstractIterator<CommandsSummary>()
         {
             Iterator<TableId> tblIt = groupByTable.keySet().iterator();
             Iterator<Map.Entry<Range, Set<RangeCommandSummary>>> rangeIt;
 
             @Override
-            protected DomainInfo computeNext()
+            protected CommandsSummary computeNext()
             {
                 while (true)
                 {
@@ -454,20 +381,20 @@ public class CommandsForRanges
     {
         TokenKey start = (TokenKey) interval.min;
         TokenKey end = (TokenKey) interval.max;
-        // TODO (correctness) : accord doesn't support wrap around, so decreaseSlightly may fail in some cases
-        // TODO (correctness) : this logic is mostly used for testing, so is it actually safe for all partitioners?
+        // TODO (required, correctness) : accord doesn't support wrap around, so decreaseSlightly may fail in some cases
+        // TODO (required, correctness) : this logic is mostly used for testing, so is it actually safe for all partitioners?
         return new TokenRange(start.withToken(start.token().decreaseSlightly()), end);
     }
 
     @Nullable
-    public DomainInfo search(Range range)
+    public CommandsSummary search(Range range)
     {
         List<RangeCommandSummary> matches = rangesToCommands.search(Interval.create(normalize(range.start(), range.startInclusive(), true),
                                                                                     normalize(range.end(), range.endInclusive(), false)));
         return result(range, matches);
     }
 
-    private DomainInfo result(Seekable seekable, Collection<RangeCommandSummary> matches)
+    private CommandsSummary result(Seekable seekable, Collection<RangeCommandSummary> matches)
     {
         if (matches.isEmpty())
             return null;
@@ -499,18 +426,20 @@ public class CommandsForRanges
             switch (ak.kindOfRoutingKey())
             {
                 case SENTINEL:
-                    key = ak.asSentinelKey().toTokenKey();
+                    // TODO (required, correctness): this doesn't work
+                    key = ak.asSentinelKey().toTokenKeyBroken();
                     continue;
                 case TOKEN:
                     TokenKey tk = ak.asTokenKey();
-                    return tk.withToken(upOrDown ? tk.token().nextValidToken() : tk.token().decreaseSlightly());
+                    // TODO (required, correctness): this doesn't work for ordered partitioner
+                    return tk.withToken(upOrDown ? tk.token().increaseSlightly() : tk.token().decreaseSlightly());
                 default:
                     throw new IllegalArgumentException("Unknown kind: " + ak.kindOfRoutingKey());
             }
         }
     }
 
-    private static class Holder implements DomainInfo
+    private static class Holder implements CommandsSummary
     {
         private final Seekable keyOrRange;
         private final Collection<RangeCommandSummary> matches;
@@ -522,27 +451,95 @@ public class CommandsForRanges
         }
 
         @Override
-        public CommandTimeseries<?> commands()
+        public <P1, T> T mapReduceFull(TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
         {
-            return build();
+            return mapReduce(testTxnId, testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, accumulate);
         }
 
         @Override
-        public Timestamp max()
+        public <P1, T> T mapReduceActive(Timestamp startedBefore, Txn.Kind.Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
         {
-            return commands().maxTimestamp();
+            return mapReduce(startedBefore, null, testKind, STARTED_BEFORE, ANY_DEPS, ANY_STATUS, map, p1, accumulate);
         }
 
-        private CommandTimeseries<?> build()
+        private <P1, T> T mapReduce(@Nonnull Timestamp testTimestamp, @Nullable TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
         {
-            CommandTimeseries.Builder<RangeCommandSummary> builder = new CommandTimeseries.Builder<>(keyOrRange, RangeCommandSummaryLoader.INSTANCE);
-            for (RangeCommandSummary m : matches)
+            // TODO (required): reconsider how we build this, to avoid having to provide range keys in order (or ensure our range search does this for us)
+            Map<Range, List<RangeCommandSummary>> collect = new TreeMap<>(Range::compare);
+            matches.forEach((summary -> {
+                if (summary.status.compareTo(SaveStatus.Erased) >= 0)
+                    return;
+
+                switch (testStartedAt)
+                {
+                    default: throw new AssertionError();
+                    case STARTED_AFTER:
+                        if (summary.txnId.compareTo(testTimestamp) <= 0) return;
+                        else break;
+                    case STARTED_BEFORE:
+                        if (summary.txnId.compareTo(testTimestamp) >= 0) return;
+                    case ANY:
+                        if (testDep != ANY_DEPS && (summary.executeAt == null || summary.executeAt.compareTo(testTxnId) < 0))
+                            return;
+                }
+
+                switch (testStatus)
+                {
+                    default: throw new AssertionError("Unhandled TestStatus: " + testStatus);
+                    case ANY_STATUS:
+                        break;
+                    case IS_PROPOSED:
+                        switch (summary.status)
+                        {
+                            default: return;
+                            case PreCommitted:
+                            case Committed:
+                            case Accepted:
+                        }
+                        break;
+                    case IS_STABLE:
+                        if (!summary.status.hasBeen(Stable) || summary.status.hasBeen(Truncated))
+                            return;
+                }
+
+                if (!testKind.test(summary.txnId.kind()))
+                    return;
+
+                if (testDep != ANY_DEPS)
+                {
+                    if (!summary.status.known.deps.hasProposedOrDecidedDeps())
+                        return;
+
+                    // TODO (required): we must ensure these txnId are limited to those we intersect in this command store
+                    // We are looking for transactions A that have (or have not) B as a dependency.
+                    // If B covers ranges [1..3] and A covers [2..3], but the command store only covers ranges [1..2],
+                    // we could have A adopt B as a dependency on [3..3] only, and have that A intersects B on this
+                    // command store, but also that there is no dependency relation between them on the overlapping
+                    // key range [2..2].
+
+                    // This can lead to problems on recovery, where we believe a transaction is a dependency
+                    // and so it is safe to execute, when in fact it is only a dependency on a different shard
+                    // (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
+                    // TODO (required): consider this some more
+                    if ((testDep == WITH) == !summary.deps.contains(testTxnId))
+                        return;
+                }
+
+                // TODO (required): ensure we are excluding any ranges that are now shard-redundant (not sure if this is enforced yet)
+                for (Range range : summary.ranges)
+                    collect.computeIfAbsent(range, ignore -> new ArrayList<>()).add(summary);
+            }));
+
+            for (Map.Entry<Range, List<RangeCommandSummary>> e : collect.entrySet())
             {
-                if (m.status == SaveStatus.Invalidated)
-                    continue;
-                builder.add(m.txnId, m);
+                for (RangeCommandSummary command : e.getValue())
+                {
+                    T initial = accumulate;
+                    accumulate = map.apply(p1, e.getKey(), command.txnId, command.executeAt, initial);
+                }
             }
-            return builder.build();
+
+            return accumulate;
         }
 
         @Override

src/java/org/apache/cassandra/service/accord/TokenRange.java

@@ -73,7 +73,7 @@ public class TokenRange extends Range.EndInclusive
     {
         RoutingKey pick = super.someIntersectingRoutingKey(ranges);
         if (pick instanceof SentinelKey)
-            pick = ((SentinelKey) pick).toTokenKey();
+            pick = ((SentinelKey) pick).toTokenKeyBroken();
         return pick;
     }
 

src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java

@@ -131,12 +131,12 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
             return new SentinelKey(table, false);
         }
 
-        public TokenKey toTokenKey()
+        public TokenKey toTokenKeyBroken()
         {
             IPartitioner partitioner = getPartitioner();
             return new TokenKey(table, isMin ?
                                          partitioner.getMinimumToken().nextValidToken() :
-                                         partitioner.getMaximumToken().decreaseSlightly());
+                                         partitioner.getMaximumTokenForSplitting().decreaseSlightly());
         }
 
         @Override

src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java

@@ -116,6 +116,7 @@ public class AsyncLoader
             case FAILED_TO_SAVE:
                 break;
             case FAILED_TO_LOAD:
+                // TODO (required): if this triggers, we trigger some other illegal state in cache management
                 throw new RuntimeException(safeRef.failure());
         }
     }
@@ -124,19 +125,18 @@ public class AsyncLoader
                                                  AsyncOperation.Context context,
                                                  List<AsyncChain<?>> listenChains)
     {
-        referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains);
         // recovery operations also need the deps data for their preaccept logic
         switch (keyHistory)
         {
-            case ALL:
-                referenceAndAssembleReadsForKey(key, context.allCommandsForKeys, commandStore.allCommandsForKeyCache(), listenChains);
-            case DEPS:
-                referenceAndAssembleReadsForKey(key, context.depsCommandsForKeys, commandStore.depsCommandsForKeyCache(), listenChains);
+            case TIMESTAMPS:
+                referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains);
+                break;
+            case COMMANDS:
+                referenceAndAssembleReadsForKey(key, context.commandsForKey, commandStore.commandsForKeyCache(), listenChains);
             case NONE:
                 break;
             default: throw new IllegalArgumentException("Unhandled keyhistory: " + keyHistory);
         }
-        referenceAndAssembleReadsForKey(key, context.updatesForKeys, commandStore.updatesForKeyCache(), listenChains);
     }
 
     private <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReads(Iterable<? extends K> keys,
@@ -195,7 +195,7 @@ public class AsyncLoader
 
     private AsyncChain<Set<PartitionKey>> findOverlappingKeys(Range range)
     {
-        Set<PartitionKey> cached = commandStore.depsCommandsForKeyCache().stream()
+        Set<PartitionKey> cached = commandStore.commandsForKeyCache().stream()
                                                .map(n -> (PartitionKey) n.key())
                                                .filter(range::contains)
                                                .collect(Collectors.toSet());
@@ -214,7 +214,7 @@ public class AsyncLoader
         if (start instanceof TokenKey)
             return (TokenKey) start;
         if (start instanceof AccordRoutingKey.SentinelKey)
-            return ((AccordRoutingKey.SentinelKey) start).toTokenKey();
+            return ((AccordRoutingKey.SentinelKey) start).toTokenKeyBroken();
         throw new IllegalArgumentException(String.format("Unable to convert RoutingKey %s (type %s) to TokenKey", start, start.getClass()));
     }
 

src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java

@@ -42,7 +42,6 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordSafeCommand;
 import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
 import org.apache.cassandra.service.accord.AccordSafeCommandStore;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForKeyUpdate;
 import org.apache.cassandra.service.accord.AccordSafeState;
 import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
 
@@ -69,26 +68,20 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
     {
         final HashMap<TxnId, AccordSafeCommand> commands = new HashMap<>();
         final TreeMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey = new TreeMap<>();
-        final TreeMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys = new TreeMap<>();
-        final TreeMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys = new TreeMap<>();
-        final TreeMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys = new TreeMap<>();
+        final TreeMap<RoutableKey, AccordSafeCommandsForKey> commandsForKey = new TreeMap<>();
 
         void releaseResources(AccordCommandStore commandStore)
         {
             commands.values().forEach(commandStore.commandCache()::release);
             timestampsForKey.values().forEach(commandStore.timestampsForKeyCache()::release);
-            depsCommandsForKeys.values().forEach(commandStore.depsCommandsForKeyCache()::release);
-            allCommandsForKeys.values().forEach(commandStore.allCommandsForKeyCache()::release);
-            updatesForKeys.values().forEach(commandStore.updatesForKeyCache()::release);
+            commandsForKey.values().forEach(commandStore.commandsForKeyCache()::release);
         }
 
         void revertChanges()
         {
             commands.values().forEach(AccordSafeState::revert);
             timestampsForKey.values().forEach(AccordSafeState::revert);
-            depsCommandsForKeys.values().forEach(AccordSafeState::revert);
-            allCommandsForKeys.values().forEach(AccordSafeState::revert);
-            updatesForKeys.values().forEach(AccordSafeState::revert);
+            commandsForKey.values().forEach(AccordSafeState::revert);
         }
     }
 
@@ -261,11 +254,11 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
                     return;
                 state(PREPARING);
             case PREPARING:
-                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.depsCommandsForKeys, context.allCommandsForKeys, context.updatesForKeys);
+                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.commandsForKey);
                 state(RUNNING);
             case RUNNING:
                 result = apply(safeStore);
-                safeStore.postExecute(context.commands, context.timestampsForKey, context.depsCommandsForKeys, context.allCommandsForKeys, context.updatesForKeys);
+                safeStore.postExecute(context.commands, context.timestampsForKey, context.commandsForKey);
                 context.releaseResources(commandStore);
                 commandStore.completeOperation(safeStore);
                 commandStore.executionOrder().unregister(this);

src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.interop;
+
+import java.util.function.BiConsumer;
+
+import accord.api.Result;
+import accord.api.Update;
+import accord.coordinate.CoordinationAdapter;
+import accord.coordinate.CoordinationAdapter.Adapters.AbstractTxnAdapter;
+import accord.coordinate.ExecutePath;
+import accord.coordinate.PersistTxn;
+import accord.local.Node;
+import accord.messages.Apply;
+import accord.primitives.Deps;
+import accord.primitives.FullRoute;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.primitives.Writes;
+import accord.topology.Topologies;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.service.accord.AccordEndpointMapper;
+import org.apache.cassandra.service.accord.api.AccordAgent;
+import org.apache.cassandra.service.accord.interop.AccordInteropExecution.InteropExecutor;
+import org.apache.cassandra.service.accord.txn.AccordUpdate;
+import org.apache.cassandra.service.accord.txn.TxnRead;
+
+import static accord.messages.Apply.Kind.Maximal;
+import static accord.messages.Apply.Kind.Minimal;
+
+public class AccordInteropAdapter extends AbstractTxnAdapter
+{
+    public static final class AccordInteropFactory implements CoordinationAdapter.Factory
+    {
+        final AccordInteropAdapter standard, recovery;
+
+        public AccordInteropFactory(AccordAgent agent, AccordEndpointMapper endpointMapper)
+        {
+            final InteropExecutor executor = new InteropExecutor(agent);
+            standard = new AccordInteropAdapter(executor, endpointMapper, Minimal);
+            recovery = new AccordInteropAdapter(executor, endpointMapper, Maximal);
+        }
+
+        @Override
+        public <R> CoordinationAdapter<R> get(TxnId txnId, Step step)
+        {
+            return (CoordinationAdapter<R>) (step == Step.InitiateRecovery ? recovery : standard);
+        }
+    };
+
+    private final InteropExecutor executor;
+    private final AccordEndpointMapper endpointMapper;
+    private final Apply.Kind applyKind;
+
+    private AccordInteropAdapter(InteropExecutor executor, AccordEndpointMapper endpointMapper, Apply.Kind applyKind)
+    {
+        this.executor = executor;
+        this.endpointMapper = endpointMapper;
+        this.applyKind = applyKind;
+    }
+
+    @Override
+    public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+    {
+        if (!doInteropExecute(node, route, txnId, txn, executeAt, deps, callback))
+            super.execute(node, all, route, path, txnId, txn, executeAt, deps, callback);
+    }
+
+    @Override
+    public void persist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+    {
+        if (applyKind == Minimal && doInteropPersist(node, all, route, txnId, txn, executeAt, deps, writes, result, callback))
+            return;
+
+        if (callback != null) callback.accept(result, null);
+        new PersistTxn(node, all, txnId, route, txn, executeAt, deps, writes, result)
+            .start(Apply.FACTORY, applyKind, all, writes, result);
+    }
+
+    private boolean doInteropExecute(Node node, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
+    {
+        // Unrecoverable repair always needs to be run by AccordInteropExecution
+        AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update());
+        ConsistencyLevel consistencyLevel = txn.read() instanceof TxnRead ? ((TxnRead) txn.read()).cassandraConsistencyLevel() : null;
+        if (updateKind != AccordUpdate.Kind.UNRECOVERABLE_REPAIR && (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE || txn.read().keys().isEmpty()))
+            return false;
+
+        new AccordInteropExecution(node, txnId, txn, updateKind, route, txn.read().keys().toParticipants(), executeAt, deps, callback, executor, consistencyLevel, endpointMapper)
+            .start();
+        return true;
+    }
+
+    private static boolean doInteropPersist(Node node, Topologies all, FullRoute<?> route, TxnId txnId, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, BiConsumer<? super Result, Throwable> callback)
+    {
+        Update update = txn.update();
+        ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null;
+        if (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ANY || writes.isEmpty())
+            return false;
+
+        new AccordInteropPersist(node, all, txnId, route, txn, executeAt, deps, writes, result, consistencyLevel, callback)
+            .start(AccordInteropApply.FACTORY, Minimal, all, writes, result);
+        return true;
+    }
+}

src/java/org/apache/cassandra/service/accord/interop/AccordInteropApply.java

@@ -31,11 +31,11 @@ import accord.local.Status;
 import accord.messages.Apply;
 import accord.messages.MessageType;
 import accord.primitives.Deps;
+import accord.primitives.FullRoute;
 import accord.primitives.Keys;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
 import accord.primitives.PartialTxn;
-import accord.primitives.Route;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
@@ -63,37 +63,37 @@ public class AccordInteropApply extends Apply implements Command.TransientListen
     public static final Apply.Factory FACTORY = new Apply.Factory()
     {
         @Override
-        public Apply create(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+        public Apply create(Kind kind, Id to, Topologies participates, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
         {
             checkArgument(kind != Kind.Maximal, "Shouldn't need to send a maximal commit with interop support");
             ConsistencyLevel commitCL = txn.update() instanceof AccordUpdate ? ((AccordUpdate) txn.update()).cassandraCommitCL() : null;
             // Any asynchronous apply option should use the regular Apply that doesn't wait for writes to complete
             if (commitCL == null || commitCL == ConsistencyLevel.ANY)
-                return Apply.FACTORY.create(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result);
-            return new AccordInteropApply(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result);
+                return Apply.FACTORY.create(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
+            return new AccordInteropApply(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
         }
     };
 
     public static final IVersionedSerializer<AccordInteropApply> serializer = new ApplySerializer<AccordInteropApply>()
     {
         @Override
-        protected AccordInteropApply deserializeApply(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Apply.Kind kind, Seekables<?, ?> keys, Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result)
+        protected AccordInteropApply deserializeApply(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Apply.Kind kind, Seekables<?, ?> keys, Timestamp executeAt, PartialDeps deps, PartialTxn txn, @Nullable FullRoute<?> fullRoute, Writes writes, Result result)
         {
-            return new AccordInteropApply(kind, txnId, scope, waitForEpoch, keys, executeAt, deps, txn, writes, result);
+            return new AccordInteropApply(kind, txnId, scope, waitForEpoch, keys, executeAt, deps, txn, fullRoute, writes, result);
         }
     };
 
     transient BitSet waitingOn;
     transient int waitingOnCount;
 
-    private AccordInteropApply(Kind kind, TxnId txnId, PartialRoute<?> route, long waitForEpoch, Seekables<?, ?> keys, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, Writes writes, Result result)
+    private AccordInteropApply(Kind kind, TxnId txnId, PartialRoute<?> route, long waitForEpoch, Seekables<?, ?> keys, Timestamp executeAt, PartialDeps deps, @Nullable PartialTxn txn, @Nullable FullRoute<?> fullRoute, Writes writes, Result result)
     {
-        super(kind, txnId, route, waitForEpoch, keys, executeAt, deps, txn, writes, result);
+        super(kind, txnId, route, waitForEpoch, keys, executeAt, deps, txn, fullRoute, writes, result);
     }
 
-    private AccordInteropApply(Kind kind, Id to, Topologies participates, Topologies executes, TxnId txnId, Route<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
+    private AccordInteropApply(Kind kind, Id to, Topologies participates, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
     {
-        super(kind, to, participates, executes, txnId, route, txn, executeAt, deps, writes, result);
+        super(kind, to, participates, txnId, route, txn, executeAt, deps, writes, result);
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/interop/AccordInteropCommit.java

@@ -31,6 +31,7 @@ import accord.primitives.FullRoute;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
 import accord.primitives.PartialTxn;
+import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
@@ -45,15 +46,15 @@ public class AccordInteropCommit extends Commit
     public static final IVersionedSerializer<AccordInteropCommit> serializer = new CommitSerializer<AccordInteropCommit, AccordInteropRead>(AccordInteropRead.class, AccordInteropRead.requestSerializer)
     {
         @Override
-        protected AccordInteropCommit deserializeCommit(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Kind kind, Ballot ballot, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nullable ReadData read)
+        protected AccordInteropCommit deserializeCommit(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Kind kind, Ballot ballot, Timestamp executeAt, Seekables<?, ?> keys, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nullable ReadData read)
         {
-            return new AccordInteropCommit(kind, txnId, scope, waitForEpoch, ballot, executeAt, partialTxn, partialDeps, fullRoute, read);
+            return new AccordInteropCommit(kind, txnId, scope, waitForEpoch, ballot, executeAt, keys, partialTxn, partialDeps, fullRoute, read);
         }
     };
 
-    public AccordInteropCommit(Kind kind, TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Ballot ballot, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nonnull ReadData readData)
+    public AccordInteropCommit(Kind kind, TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Ballot ballot, Timestamp executeAt, Seekables<?, ?> keys, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nonnull ReadData readData)
     {
-        super(kind, txnId, scope, waitForEpoch, ballot, executeAt, partialTxn, partialDeps, fullRoute, readData);
+        super(kind, txnId, scope, waitForEpoch, ballot, executeAt, keys, partialTxn, partialDeps, fullRoute, readData);
     }
 
     public AccordInteropCommit(Kind kind, Node.Id to, Topology coordinateTopology, Topologies topologies, TxnId txnId, Txn txn, FullRoute<?> route, Timestamp executeAt, Deps deps, AccordInteropRead read)

src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java

@@ -37,9 +37,6 @@ import org.slf4j.LoggerFactory;
 import accord.api.Agent;
 import accord.api.Data;
 import accord.api.Result;
-import accord.coordinate.Execute;
-import accord.coordinate.Persist;
-import accord.coordinate.ExecuteTxn;
 import accord.local.AgentExecutor;
 import accord.local.CommandStore;
 import accord.local.Node;
@@ -95,6 +92,8 @@ import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Dispatcher;
 
+import static accord.coordinate.CoordinationAdapter.Factory.Step.Continue;
+import static accord.coordinate.CoordinationAdapter.Invoke.persist;
 import static accord.utils.Invariants.checkArgument;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
@@ -110,11 +109,11 @@ import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWri
  * on its inputs.
  *
  */
-public class AccordInteropExecution implements Execute, ReadCoordinator, MaximalCommitSender
+public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSender
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordInteropExecution.class);
 
-    private static class InteropExecutor implements AgentExecutor
+    static class InteropExecutor implements AgentExecutor
     {
         private final AccordAgent agent;
 
@@ -143,29 +142,6 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
         }
     }
 
-    public static class Factory implements Execute.Factory
-    {
-        private final InteropExecutor executor;
-        private final AccordEndpointMapper endpointMapper;
-
-        public Factory(AccordAgent agent, AccordEndpointMapper endpointMapper)
-        {
-            this.executor = new InteropExecutor(agent);
-            this.endpointMapper = endpointMapper;
-        }
-
-        @Override
-        public Execute create(Node node, Topologies topologies, Path path, TxnId txnId, Txn txn, FullRoute<?> route, Participants<?> readScope, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback)
-        {
-            // Unrecoverable repair always needs to be run by AccordInteropExecution
-            AccordUpdate.Kind updateKind = AccordUpdate.kind(txn.update());
-            ConsistencyLevel consistencyLevel = txn.read() instanceof TxnRead ? ((TxnRead) txn.read()).cassandraConsistencyLevel() : null;
-            if (updateKind != AccordUpdate.Kind.UNRECOVERABLE_REPAIR && (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE || txn.read().keys().isEmpty()))
-                return ExecuteTxn.FACTORY.create(node, topologies, path, txnId, txn, route, readScope, executeAt, deps, callback);
-            return new AccordInteropExecution(node, txnId, txn, updateKind, route, readScope, executeAt, deps, callback, executor, consistencyLevel, endpointMapper);
-        }
-    }
-
     private final Node node;
     private final TxnId txnId;
     private final Txn txn;
@@ -253,7 +229,7 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
     {
         Node.Id id = endpointMapper.mappedId(to);
         SinglePartitionReadCommand command = (SinglePartitionReadCommand) message.payload;
-        AccordInteropRead read = new AccordInteropRead(id, executes, txnId, readScope, executeAt, command);
+        AccordInteropRead read = new AccordInteropRead(id, executes, txnId, readScope, executeAt.epoch(), command);
         // TODO (required): understand interop and whether StableFastPath is appropriate
         AccordInteropCommit commit = new AccordInteropCommit(Kind.StableFastPath, id, coordinateTopology, allTopologies,
                                                              txnId, txn, route, executeAt, deps, read);
@@ -265,7 +241,7 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
     {
         Node.Id id = endpointMapper.mappedId(to);
         Mutation mutation = message.payload;
-        AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt, mutation);
+        AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), mutation);
         node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this));
     }
 
@@ -350,7 +326,6 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
                 node.send(to, new Commit(Kind.StableFastPath, to, coordinateTopology, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps, (ReadTxnData) null));
     }
 
-    @Override
     public void start()
     {
         if (coordinateTopology != executeTopology)
@@ -370,7 +345,7 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
         CommandStore cs = node.commandStores().select(route.homeKey());
         result.beginAsResult().withExecutor(cs).begin((data, failure) -> {
             if (failure == null)
-                Persist.persist(node, executes, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, data), txn.result(txnId, executeAt, data), callback);
+                persist(node.coordinationAdapter(txnId, Continue), node, executes, route, txnId, txn, executeAt, deps, txn.execute(txnId, executeAt, data), txn.result(txnId, executeAt, data), callback);
             else
                 callback.accept(null, failure);
         });

src/java/org/apache/cassandra/service/accord/interop/AccordInteropPersist.java

@@ -21,9 +21,7 @@ package org.apache.cassandra.service.accord.interop;
 import java.util.function.BiConsumer;
 
 import accord.api.Result;
-import accord.api.Update;
 import accord.coordinate.Persist;
-import accord.coordinate.PersistTxn;
 import accord.coordinate.tracking.AppliedTracker;
 import accord.coordinate.tracking.QuorumTracker;
 import accord.coordinate.tracking.RequestStatus;
@@ -39,7 +37,6 @@ import accord.primitives.Writes;
 import accord.topology.Topologies;
 import accord.utils.Invariants;
 import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.utils.Throwables;
 
 /**
@@ -49,19 +46,6 @@ import org.apache.cassandra.utils.Throwables;
  */
 public class AccordInteropPersist extends Persist
 {
-    public static Persist.Factory FACTORY = new Persist.Factory()
-    {
-        @Override
-        public Persist create(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result)
-        {
-            Update update = txn.update();
-            ConsistencyLevel consistencyLevel = update instanceof AccordUpdate ? ((AccordUpdate) update).cassandraCommitCL() : null;
-            if (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ANY || writes.isEmpty())
-                return PersistTxn.FACTORY.create(node, topologies, txnId, route, txn, executeAt, deps, writes, result);
-            return new AccordInteropPersist(node, topologies, txnId, route, txn, executeAt, deps, writes, result, consistencyLevel);
-        }
-    };
-
     private static class CallbackHolder
     {
         private final ResponseTracker tracker;
@@ -97,7 +81,6 @@ public class AccordInteropPersist extends Persist
             handleStatus(tracker.recordSuccess(node));
         }
 
-
         public void recordFailure(Node.Id node, Throwable throwable)
         {
             failure = Throwables.merge(failure, throwable);
@@ -106,29 +89,28 @@ public class AccordInteropPersist extends Persist
     }
 
     private final ConsistencyLevel consistencyLevel;
-    private CallbackHolder holder = null;
+    private CallbackHolder callback;
 
-    public AccordInteropPersist(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, ConsistencyLevel consistencyLevel)
+    public AccordInteropPersist(Node node, Topologies topologies, TxnId txnId, FullRoute<?> route, Txn txn, Timestamp executeAt, Deps deps, Writes writes, Result result, ConsistencyLevel consistencyLevel, BiConsumer<? super Result, Throwable> clientCallback)
     {
         super(node, topologies, txnId, route, txn, executeAt, deps, writes, result);
         Invariants.checkArgument(consistencyLevel == ConsistencyLevel.QUORUM || consistencyLevel == ConsistencyLevel.ALL || consistencyLevel == ConsistencyLevel.SERIAL || consistencyLevel == ConsistencyLevel.ONE);
         this.consistencyLevel = consistencyLevel;
+        registerClientCallback(result, clientCallback);
     }
 
-    @Override
-    public void registerClientCallback(Writes writes, Result result, BiConsumer<? super Result, Throwable> clientCallback)
+    public void registerClientCallback(Result result, BiConsumer<? super Result, Throwable> clientCallback)
     {
-
-        Invariants.checkState(holder == null);
+        Invariants.checkState(callback == null);
         switch (consistencyLevel)
         {
             case ONE: // Can safely upgrade ONE to QUORUM/SERIAL to get a synchronous commit
             case SERIAL:
             case QUORUM:
-                holder = new CallbackHolder(new QuorumTracker(topologies), result, clientCallback);
+                callback = new CallbackHolder(new QuorumTracker(topologies), result, clientCallback);
                 break;
             case ALL:
-                holder = new CallbackHolder(new AppliedTracker(topologies), result, clientCallback);
+                callback = new CallbackHolder(new AppliedTracker(topologies), result, clientCallback);
                 break;
             default:
                 throw new IllegalArgumentException("Unhandled consistency level: " + consistencyLevel);
@@ -143,7 +125,7 @@ public class AccordInteropPersist extends Persist
         {
             case Redundant:
             case Applied:
-                holder.recordSuccess(from);
+                callback.recordSuccess(from);
                 return;
             case Insufficient:
                 // On insufficient Persist will send a commit with the missing information
@@ -156,12 +138,12 @@ public class AccordInteropPersist extends Persist
     @Override
     public void onFailure(Node.Id from, Throwable failure)
     {
-        holder.recordFailure(from, failure);
+        callback.recordFailure(from, failure);
     }
 
     @Override
     public void onCallbackFailure(Node.Id from, Throwable failure)
     {
-        holder.recordFailure(from, failure);
+        callback.recordFailure(from, failure);
     }
 }

src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java

@@ -24,7 +24,7 @@ import javax.annotation.Nullable;
 import accord.api.Data;
 import accord.local.Node;
 import accord.local.SafeCommandStore;
-import accord.messages.AbstractExecute;
+import accord.messages.ReadData;
 import accord.messages.MessageType;
 import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
@@ -51,7 +51,10 @@ import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializers.ReadDataSerializer;
 
-public class AccordInteropRead extends AbstractExecute
+import static accord.local.SaveStatus.PreApplied;
+import static accord.local.SaveStatus.ReadyToExecute;
+
+public class AccordInteropRead extends ReadData
 {
     public static final IVersionedSerializer<AccordInteropRead> requestSerializer = new ReadDataSerializer<AccordInteropRead>()
     {
@@ -60,8 +63,7 @@ public class AccordInteropRead extends AbstractExecute
         {
             CommandSerializers.txnId.serialize(read.txnId, out, version);
             KeySerializers.participants.serialize(read.readScope, out, version);
-            out.writeUnsignedVInt(read.waitForEpoch());
-            out.writeUnsignedVInt(read.executeAtEpoch - read.waitForEpoch());
+            out.writeUnsignedVInt(read.executeAtEpoch);
             SinglePartitionReadCommand.serializer.serialize(read.command, out, version);
         }
 
@@ -70,10 +72,9 @@ public class AccordInteropRead extends AbstractExecute
         {
             TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
             Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
-            long waitForEpoch = in.readUnsignedVInt();
-            long executeAtEpoch = in.readUnsignedVInt() + waitForEpoch;
+            long executeAtEpoch = in.readUnsignedVInt();
             SinglePartitionReadCommand command = (SinglePartitionReadCommand) SinglePartitionReadCommand.serializer.deserialize(in, version);
-            return new AccordInteropRead(txnId, readScope, waitForEpoch, executeAtEpoch, command);
+            return new AccordInteropRead(txnId, readScope, executeAtEpoch, command);
         }
 
         @Override
@@ -81,8 +82,7 @@ public class AccordInteropRead extends AbstractExecute
         {
             return CommandSerializers.txnId.serializedSize(read.txnId, version)
                    + KeySerializers.participants.serializedSize(read.readScope, version)
-                   + TypeSizes.sizeofUnsignedVInt(read.waitForEpoch())
-                   + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch - read.waitForEpoch())
+                   + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
                    + SinglePartitionReadCommand.serializer.serializedSize(read.command, version);
         }
     };
@@ -146,31 +146,39 @@ public class AccordInteropRead extends AbstractExecute
         }
     }
 
+    private static final ExecuteOn EXECUTE_ON = new ExecuteOn(ReadyToExecute, PreApplied);
+
     private final SinglePartitionReadCommand command;
 
-    public AccordInteropRead(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> readScope, Timestamp executeAt, SinglePartitionReadCommand command)
+    public AccordInteropRead(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> readScope, long executeAtEpoch, SinglePartitionReadCommand command)
     {
-        super(to, topologies, txnId, readScope, executeAt);
+        super(to, topologies, txnId, readScope, executeAtEpoch);
         this.command = command;
     }
 
-    public AccordInteropRead(TxnId txnId, Participants<?> readScope, long executeAtEpoch, long waitForEpoch, SinglePartitionReadCommand command)
+    public AccordInteropRead(TxnId txnId, Participants<?> readScope, long executeAtEpoch, SinglePartitionReadCommand command)
     {
-        super(txnId, readScope, executeAtEpoch, waitForEpoch);
+        super(txnId, readScope, executeAtEpoch);
         this.command = command;
     }
 
     @Override
-    protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
+    public ReadType kind()
+    {
+        return ReadType.readTxnData;
+    }
+
+    @Override
+    protected AsyncChain<Data> beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
     {
         // TODO (required): subtract unavailable ranges, either from read or from response (or on coordinator)
         return AsyncChains.ofCallable(Stage.READ.executor(), () -> new LocalReadData(ReadCommandVerbHandler.instance.doRead(command, false)));
     }
 
     @Override
-    protected boolean canExecutePreApplied()
+    protected ExecuteOn executeOn()
     {
-        return true;
+        return EXECUTE_ON;
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadRepair.java

@@ -24,7 +24,8 @@ import javax.annotation.Nullable;
 import accord.api.Data;
 import accord.local.Node;
 import accord.local.SafeCommandStore;
-import accord.messages.AbstractExecute;
+import accord.local.SaveStatus;
+import accord.messages.ReadData;
 import accord.messages.MessageType;
 import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
@@ -56,7 +57,7 @@ import org.apache.cassandra.service.accord.serializers.ReadDataSerializers.ReadD
  * ensuring that the contents of the read repair consist of data that isn't from transactions that
  * haven't been committed yet at this command store.
  */
-public class AccordInteropReadRepair extends AbstractExecute
+public class AccordInteropReadRepair extends ReadData
 {
     public static final IVersionedSerializer<AccordInteropReadRepair> requestSerializer = new ReadDataSerializer<AccordInteropReadRepair>()
     {
@@ -65,8 +66,7 @@ public class AccordInteropReadRepair extends AbstractExecute
         {
             CommandSerializers.txnId.serialize(repair.txnId, out, version);
             KeySerializers.participants.serialize(repair.readScope, out, version);
-            out.writeUnsignedVInt(repair.waitForEpoch());
-            out.writeUnsignedVInt(repair.executeAtEpoch - repair.waitForEpoch());
+            out.writeUnsignedVInt(repair.executeAtEpoch);
             Mutation.serializer.serialize(repair.mutation, out, version);
         }
 
@@ -75,10 +75,9 @@ public class AccordInteropReadRepair extends AbstractExecute
         {
             TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
             Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
-            long waitForEpoch = in.readUnsignedVInt();
-            long executeAtEpoch = in.readUnsignedVInt() + waitForEpoch;
+            long executeAtEpoch = in.readUnsignedVInt();
             Mutation mutation = Mutation.serializer.deserialize(in, version);
-            return new AccordInteropReadRepair(txnId, readScope, waitForEpoch, executeAtEpoch, mutation);
+            return new AccordInteropReadRepair(txnId, readScope, executeAtEpoch, mutation);
         }
 
         @Override
@@ -86,8 +85,7 @@ public class AccordInteropReadRepair extends AbstractExecute
         {
             return CommandSerializers.txnId.serializedSize(repair.txnId, version)
                    + KeySerializers.participants.serializedSize(repair.readScope, version)
-                   + TypeSizes.sizeofUnsignedVInt(repair.waitForEpoch())
-                   + TypeSizes.sizeofUnsignedVInt(repair.executeAtEpoch - repair.waitForEpoch())
+                   + TypeSizes.sizeofUnsignedVInt(repair.executeAtEpoch)
                    + Mutation.serializer.serializedSize(repair.mutation, version);
         }
     };
@@ -106,6 +104,8 @@ public class AccordInteropReadRepair extends AbstractExecute
         }
     }
 
+    private static final ExecuteOn EXECUTE_ON = new ExecuteOn(SaveStatus.ReadyToExecute, SaveStatus.Applied);
+
     private final Mutation mutation;
 
     private static final IVersionedSerializer<Data> noop_data_serializer = new IVersionedSerializer<Data>()
@@ -120,21 +120,33 @@ public class AccordInteropReadRepair extends AbstractExecute
 
     public static final IVersionedSerializer<ReadReply> replySerializer = new ReadDataSerializers.ReplySerializer<>(noop_data_serializer);
 
-    public AccordInteropReadRepair(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> readScope, Timestamp executeAt, Mutation mutation)
+    public AccordInteropReadRepair(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> readScope, long executeAtEpoch, Mutation mutation)
     {
-        super(to, topologies, txnId, readScope, executeAt);
+        super(to, topologies, txnId, readScope, executeAtEpoch);
         this.mutation = mutation;
     }
 
-    public AccordInteropReadRepair(TxnId txnId, Participants<?> readScope, long executeAtEpoch, long waitForEpoch, Mutation mutation)
+    public AccordInteropReadRepair(TxnId txnId, Participants<?> readScope, long executeAtEpoch, Mutation mutation)
     {
         // TODO (review): remove followup read - Is there anything left to be done for this or can I remove it?
-        super(txnId, readScope, executeAtEpoch, waitForEpoch);
+        super(txnId, readScope, executeAtEpoch);
         this.mutation = mutation;
     }
 
     @Override
-    protected AsyncChain<Data> execute(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
+    protected ExecuteOn executeOn()
+    {
+        return EXECUTE_ON;
+    }
+
+    @Override
+    public ReadType kind()
+    {
+        return ReadType.readTxnData;
+    }
+
+    @Override
+    protected AsyncChain<Data> beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
     {
         // TODO (required): subtract unavailable ranges, either from read or from response (or on coordinator)
         return AsyncChains.ofCallable(Verb.READ_REPAIR_REQ.stage.executor(), () -> {
@@ -143,18 +155,6 @@ public class AccordInteropReadRepair extends AbstractExecute
                                       });
     }
 
-    @Override
-    protected boolean canExecutePreApplied()
-    {
-        return true;
-    }
-
-    @Override
-    protected boolean executeIfObsoleted()
-    {
-        return true;
-    }
-
     @Override
     protected ReadOk constructReadOk(Ranges unavailable, Data data)
     {

src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import accord.api.Result;
 import accord.messages.Apply;
+import accord.primitives.FullRoute;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
 import accord.primitives.PartialTxn;
@@ -68,11 +69,12 @@ public class ApplySerializers
             CommandSerializers.timestamp.serialize(apply.executeAt, out, version);
             DepsSerializer.partialDeps.serialize(apply.deps, out, version);
             CommandSerializers.nullablePartialTxn.serialize(apply.txn, out, version);
+            KeySerializers.nullableFullRoute.serialize(apply.fullRoute, out, version);
             CommandSerializers.writes.serialize(apply.writes, out, version);
         }
 
         protected abstract A deserializeApply(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Apply.Kind kind, Seekables<?, ?> keys,
-                                              Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result);
+                                              Timestamp executeAt, PartialDeps deps, PartialTxn txn, FullRoute<?> fullRoute, Writes writes, Result result);
 
         @Override
         public A deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch) throws IOException
@@ -83,6 +85,7 @@ public class ApplySerializers
                                     CommandSerializers.timestamp.deserialize(in, version),
                                     DepsSerializer.partialDeps.deserialize(in, version),
                                     CommandSerializers.nullablePartialTxn.deserialize(in, version),
+                                    KeySerializers.nullableFullRoute.deserialize(in, version),
                                     CommandSerializers.writes.deserialize(in, version),
                                     CommandSerializers.APPLIED);
         }
@@ -95,6 +98,7 @@ public class ApplySerializers
                    + CommandSerializers.timestamp.serializedSize(apply.executeAt, version)
                    + DepsSerializer.partialDeps.serializedSize(apply.deps, version)
                    + CommandSerializers.nullablePartialTxn.serializedSize(apply.txn, version)
+                   + KeySerializers.nullableFullRoute.serializedSize(apply.fullRoute, version)
                    + CommandSerializers.writes.serializedSize(apply.writes, version);
         }
     }
@@ -103,9 +107,9 @@ public class ApplySerializers
     {
         @Override
         protected Apply deserializeApply(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Apply.Kind kind, Seekables<?, ?> keys,
-                               Timestamp executeAt, PartialDeps deps, PartialTxn txn, Writes writes, Result result)
+                               Timestamp executeAt, PartialDeps deps, PartialTxn txn, FullRoute<?> fullRoute, Writes writes, Result result)
         {
-            return Apply.SerializationSupport.create(txnId, scope, waitForEpoch, kind, keys, executeAt, deps, txn, writes, result);
+            return Apply.SerializationSupport.create(txnId, scope, waitForEpoch, kind, keys, executeAt, deps, txn, fullRoute, writes, result);
         }
     };
 

src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java

@@ -46,6 +46,8 @@ import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.serializers.IVersionedWithKeysSerializer.AbstractWithKeysSerializer;
+import org.apache.cassandra.service.accord.serializers.IVersionedWithKeysSerializer.NullableWithKeysSerializer;
 import org.apache.cassandra.service.accord.serializers.SmallEnumSerializer.NullableSmallEnumSerializer;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
@@ -140,7 +142,7 @@ public class CommandSerializers
         }
     }
 
-    public static class PartialTxnSerializer implements IVersionedSerializer<PartialTxn>
+    public static class PartialTxnSerializer extends AbstractWithKeysSerializer implements IVersionedWithKeysSerializer<Seekables<?, ?>, PartialTxn>
     {
         private final IVersionedSerializer<Read> readSerializer;
         private final IVersionedSerializer<Query> querySerializer;
@@ -155,10 +157,52 @@ public class CommandSerializers
 
         @Override
         public void serialize(PartialTxn txn, DataOutputPlus out, int version) throws IOException
+        {
+            KeySerializers.seekables.serialize(txn.keys(), out, version);
+            serializeWithoutKeys(txn, out, version);
+        }
+
+        @Override
+        public PartialTxn deserialize(DataInputPlus in, int version) throws IOException
+        {
+            Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
+            return deserializeWithoutKeys(keys, in, version);
+        }
+
+        @Override
+        public long serializedSize(PartialTxn txn, int version)
+        {
+            long size = KeySerializers.seekables.serializedSize(txn.keys(), version);
+            size += serializedSizeWithoutKeys(txn, version);
+            return size;
+        }
+
+        @Override
+        public void serialize(Seekables<?, ?> superset, PartialTxn txn, DataOutputPlus out, int version) throws IOException
+        {
+            serializeSubset(txn.keys(), superset, out);
+            serializeWithoutKeys(txn, out, version);
+        }
+
+        @Override
+        public PartialTxn deserialize(Seekables<?, ?> superset, DataInputPlus in, int version) throws IOException
+        {
+            Seekables<?, ?> keys = deserializeSubset(superset, in);
+            return deserializeWithoutKeys(keys, in, version);
+        }
+
+        @Override
+        public long serializedSize(Seekables<?, ?> superset, PartialTxn txn, int version)
+        {
+            long size = serializedSubsetSize(txn.keys(), superset);
+            size += serializedSizeWithoutKeys(txn, version);
+            return size;
+        }
+
+        private void serializeWithoutKeys(PartialTxn txn, DataOutputPlus out, int version) throws IOException
         {
             CommandSerializers.kind.serialize(txn.kind(), out, version);
             KeySerializers.ranges.serialize(txn.covering(), out, version);
-            KeySerializers.seekables.serialize(txn.keys(), out, version);
             readSerializer.serialize(txn.read(), out, version);
             querySerializer.serialize(txn.query(), out, version);
             out.writeBoolean(txn.update() != null);
@@ -166,24 +210,21 @@ public class CommandSerializers
                 updateSerializer.serialize(txn.update(), out, version);
         }
 
-        @Override
-        public PartialTxn deserialize(DataInputPlus in, int version) throws IOException
+        private PartialTxn deserializeWithoutKeys(Seekables<?, ?> keys, DataInputPlus in, int version) throws IOException
         {
             Txn.Kind kind = CommandSerializers.kind.deserialize(in, version);
             Ranges covering = KeySerializers.ranges.deserialize(in, version);
-            Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
             Read read = readSerializer.deserialize(in, version);
             Query query = querySerializer.deserialize(in, version);
             Update update = in.readBoolean() ? updateSerializer.deserialize(in, version) : null;
             return new PartialTxn.InMemory(covering, kind, keys, read, query, update);
         }
 
-        @Override
-        public long serializedSize(PartialTxn txn, int version)
+
+        private long serializedSizeWithoutKeys(PartialTxn txn, int version)
         {
             long size = CommandSerializers.kind.serializedSize(txn.kind(), version);
             size += KeySerializers.ranges.serializedSize(txn.covering(), version);
-            size += KeySerializers.seekables.serializedSize(txn.keys(), version);
             size += readSerializer.serializedSize(txn.read(), version);
             size += querySerializer.serializedSize(txn.query(), version);
             size += TypeSizes.sizeof(txn.update() != null);
@@ -197,8 +238,8 @@ public class CommandSerializers
     private static final IVersionedSerializer<Query> query = new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer);
     private static final IVersionedSerializer<Update> update = new CastingSerializer<>(AccordUpdate.class, AccordUpdate.serializer);
 
-    public static final IVersionedSerializer<PartialTxn> partialTxn = new PartialTxnSerializer(read, query, update);
-    public static final IVersionedSerializer<PartialTxn> nullablePartialTxn = NullableSerializer.wrap(partialTxn);
+    public static final IVersionedWithKeysSerializer<Seekables<?, ?>, PartialTxn> partialTxn = new PartialTxnSerializer(read, query, update);
+    public static final IVersionedWithKeysSerializer<Seekables<?, ?>, PartialTxn> nullablePartialTxn = new NullableWithKeysSerializer<>(partialTxn);
 
     public static final EnumSerializer<SaveStatus> saveStatus = new EnumSerializer<>(SaveStatus.class);
     public static final EnumSerializer<Status> status = new EnumSerializer<>(Status.class);

src/java/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializer.java

@@ -18,190 +18,832 @@
 
 package org.apache.cassandra.service.accord.serializers;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Arrays;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
 
-import accord.impl.CommandTimeseries.CommandLoader;
-import accord.local.Command;
-import accord.local.SaveStatus;
-import accord.primitives.PartialDeps;
+import accord.api.Key;
+import accord.impl.CommandsForKey;
+import accord.impl.CommandsForKey.Info;
+import accord.impl.CommandsForKey.InternalStatus;
+import accord.impl.CommandsForKey.NoInfo;
+import accord.local.Node;
+import accord.primitives.Routable.Domain;
 import accord.primitives.Timestamp;
+import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.db.marshal.ByteBufferAccessor;
-import org.apache.cassandra.db.marshal.ValueAccessor;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.LocalVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.AccordSerializerVersion;
+import org.apache.cassandra.utils.vint.VIntCoding;
+
+import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.primitives.Txn.Kind.Read;
+import static accord.primitives.Txn.Kind.Write;
+import static accord.utils.ArrayBuffers.cachedInts;
+import static accord.utils.ArrayBuffers.cachedTxnIds;
+import static org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer.TxnIdFlags.EXTENDED;
+import static org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer.TxnIdFlags.EXTENDED_BITS;
+import static org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer.TxnIdFlags.RAW;
+import static org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer.TxnIdFlags.RAW_BITS;
+import static org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer.TxnIdFlags.STANDARD;
+import static org.apache.cassandra.utils.ByteBufferUtil.readLeastSignificantBytes;
+import static org.apache.cassandra.utils.ByteBufferUtil.writeLeastSignificantBytes;
+import static org.apache.cassandra.utils.ByteBufferUtil.writeMostSignificantBytes;
 
 public class CommandsForKeySerializer
 {
-    @VisibleForTesting
-    public static final IVersionedSerializer<List<TxnId>> depsIdSerializer = new IVersionedSerializer<List<TxnId>>()
+    private static final int HAS_MISSING_DEPS_HEADER_BIT = 0x1;
+    private static final int HAS_EXECUTE_AT_HEADER_BIT = 0x2;
+    private static final int HAS_NON_STANDARD_FLAGS = 0x4;
+
+    /**
+     * We read/write a fixed number of intial bytes for each command, with an initial flexible number of flag bits
+     * and the remainder interpreted as the HLC/epoch/node.
+     *
+     * The preamble encodes:
+     *  vint32: number of commands
+     *  vint32: number of unique node Ids
+     *  [unique node ids]
+     *  two flag bytes:
+     *   bit 0 is set if there are any missing ids;
+     *   bit 1 is set if there are any executeAt specified
+     *   bit 2 is set if there are any queries present besides reads/writes
+     *   bits 3-4 number of header bytes to read for each command
+     *   bits 5-6: level 0 extra hlc bytes to read
+     *   bits 7-8: level 1 extra hlc bytes to read (+ 1 + level 0)
+     *   bits 9-10: level 2 extra hlc bytes to read (+ 1 + level 1)
+     *   bits 12-13: level 3 extra hlc bytes to read (+ 1 + level 2)
+     *
+     * In order, for each command, we consume:
+     * 3 bits for the InternalStatus of the command
+     * 1 optional bit: if the status encodes an executeAt, indicating if the executeAt is not the TxnId
+     * 1 optional bit: if the status encodes any dependencies and there are non-zero missing ids, indicating if there are any missing for this command
+     * 1 or 2 bits for the kind of the TxnId: 0=key read, 1=key write, 2=exclusive sync point,3=read 16 bits
+     * 1 bit encoding if the epoch has changed
+     * 2 optional bits: if the prior bit is set, indicating how many bits should be read for the epoch increment: 0=none (increment by 1); 1=4, 2=8, 3=32
+     * 4 option bits: if prior bits=01, epoch delta
+     * N node id bits (where 2^N unique node ids in the CFK)
+     * 2 bits indicating how many more payload bytes should be read, with mapping written in header
+     * all remaining bits are interpreted as a delta from the prior HLC
+     *
+     * if txnId kind flag is 3, read an additional 2 bytes for TxnId flag
+     * if epoch increment flag is 2 or 3, read additional 1 or 4 bytes for epoch delta
+     * if executeAt is expected, read vint32 for epoch, vint32 for delta from txnId hlc, and ceil(N/8) bytes for node id
+     *
+     * After writing all transactions, we then write out the missing txnid collections. This is written at the end
+     * so that on deserialization we have already read all of the TxnId. This also permits more efficient serialization,
+     * as we can encode a single bit stream with the optimal number of bits.
+     * TODO (desired): we could prefix this collection with the subset of TxnId that are actually missing from any other
+     *   deps, so as to shrink this collection much further.
+     */
+    // TODO (expected): offer filtering option that does not need to reconstruct objects/info, reusing prior encoding decisions
+    // TODO (expected): accept new redundantBefore on load to avoid deserializing stale data
+    // TODO (desired): determine timestamp resolution as a factor of 10
+    public static ByteBuffer toBytesWithoutKey(CommandsForKey cfk)
     {
-        @Override
-        public void serialize(List<TxnId> ids, DataOutputPlus out, int version) throws IOException
+        int commandCount = cfk.size();
+        if (commandCount == 0)
         {
-            out.writeInt(ids.size());
-            for (int i=0,mi=ids.size(); i<mi; i++)
-                CommandSerializers.txnId.serialize(ids.get(i), out, version);
+            // TODO (expected): we should not need to special-case, but best solution here is not to store redundantBefore;
+            //    but this requires some modest deeper changes, so for now special-case serialization when empty
+            Timestamp redundantBefore = cfk.redundantBefore();
+            ByteBuffer out = ByteBuffer.allocate(TypeSizes.sizeofUnsignedVInt(0) +
+                                                 TypeSizes.sizeofUnsignedVInt(redundantBefore.epoch()) +
+                                                 TypeSizes.sizeofUnsignedVInt(redundantBefore.hlc()) +
+                                                 TypeSizes.sizeofUnsignedVInt(redundantBefore.flags()) +
+                                                 TypeSizes.sizeofUnsignedVInt(redundantBefore.node.id));
+            VIntCoding.writeUnsignedVInt32(0, out);
+            VIntCoding.writeUnsignedVInt(redundantBefore.epoch(), out);
+            VIntCoding.writeUnsignedVInt(redundantBefore.hlc(), out);
+            VIntCoding.writeUnsignedVInt32(redundantBefore.flags(), out);
+            VIntCoding.writeUnsignedVInt32(redundantBefore.node.id, out);
+            out.flip();
+            return out;
         }
 
-        @Override
-        public List<TxnId> deserialize(DataInputPlus in, int version) throws IOException
+        int[] nodeIds = cachedInts().getInts(Math.min(64, commandCount));
+        try
         {
-            int size = in.readInt();
-            List<TxnId> ids = new ArrayList<>(size);
-            for (int i=0; i<size; i++)
-                ids.add(CommandSerializers.txnId.deserialize(in, version));
-            return ids;
-        }
-
-        @Override
-        public long serializedSize(List<TxnId> ids, int version)
-        {
-            long size = TypeSizes.INT_SIZE;
-            for (int i=0,mi=ids.size(); i<mi; i++)
-                size += CommandSerializers.txnId.serializedSize(ids.get(i), version);
-            return size;
-        }
-    };
-
-    private static final LocalVersionedSerializer<List<TxnId>> depsIdsLocalSerializer = new LocalVersionedSerializer<>(AccordSerializerVersion.CURRENT, AccordSerializerVersion.serializer, depsIdSerializer);
-
-    public static final CommandLoader<ByteBuffer> loader = new AccordCFKLoader();
-    private static class AccordCFKLoader implements CommandLoader<ByteBuffer>
-    {
-        private static final int HAS_DEPS = 0x01;
-        private static final int HAS_EXECUTE_AT = 0x02;
-
-        private static final long FIXED_SIZE;
-        private static final int FLAG_OFFSET;
-        private static final int STATUS_OFFSET;
-        private static final int TXNID_OFFSET;
-        private static final int EXECUTEAT_OFFSET;
-        private static final int DEPS_OFFSET;
-
-        static
-        {
-            long size = 0;
-
-            FLAG_OFFSET = (int) size;
-            size += TypeSizes.BYTE_SIZE;
-
-            STATUS_OFFSET = (int) size;
-            size += TypeSizes.BYTE_SIZE;
-
-            TXNID_OFFSET = (int) size;
-            size += CommandSerializers.txnId.serializedSize();
-
-            FIXED_SIZE = size;
-
-            EXECUTEAT_OFFSET = (int) size;
-            size += CommandSerializers.timestamp.serializedSize();
-
-            DEPS_OFFSET = (int) size;
-        }
-
-        private int serializedSize(Command command)
-        {
-            return (int) (FIXED_SIZE
-                          + (command.executeAt() != null ? CommandSerializers.timestamp.serializedSize() : 0)
-                          + (command.partialDeps() != null ? depsIdsLocalSerializer.serializedSize(command.partialDeps().txnIds()) : 0));
-        }
-
-        private static final ValueAccessor<ByteBuffer> accessor = ByteBufferAccessor.instance;
-
-        private static byte toByte(int v)
-        {
-            Invariants.checkArgument(v < Byte.MAX_VALUE, "Value %d is larger than %d", v, Byte.MAX_VALUE);
-            return (byte) v;
-        }
-
-        private AccordCFKLoader() {}
-
-        @Override
-        public ByteBuffer saveForCFK(Command command)
-        {
-            int flags = 0;
-
-            PartialDeps deps = command.partialDeps();
-            Timestamp executeAt = command.executeAt();
-            if (deps != null)
-                flags |= HAS_DEPS;
-            if (executeAt != null)
-                flags |= HAS_EXECUTE_AT;
-
-            int size = serializedSize(command);
-            ByteBuffer buffer = accessor.allocate(size);
-            accessor.putByte(buffer, FLAG_OFFSET, toByte(flags));
-            accessor.putByte(buffer, STATUS_OFFSET, toByte(command.saveStatus().ordinal()));
-            CommandSerializers.txnId.serialize(command.txnId(), buffer, accessor, TXNID_OFFSET);
-            if (executeAt != null)
-                CommandSerializers.timestamp.serialize(executeAt, buffer, accessor, EXECUTEAT_OFFSET);
-            if (deps != null)
+            // first compute the unique Node Ids and some basic characteristics of the data, such as
+            // whether we have any missing transactions to encode, any executeAt that are not equal to their TxnId
+            // and whether there are any non-standard flag bits to encode
+            boolean hasNonStandardFlags = false;
+            int nodeIdCount, missingIdCount = 0, executeAtCount = 0, bitsPerExecuteAtFlags = 0;
+            int bitsPerExecuteAtEpochDelta = 0, bitsPerExecuteAtHlcDelta = 1; // to permit us to use full 64 bits and encode in 5 bits we force at least one hlc bit
             {
-                ByteBuffer duplicate = buffer.duplicate();
-                duplicate.position(executeAt != null ? DEPS_OFFSET : EXECUTEAT_OFFSET);
-                try (DataOutputBuffer out = new DataOutputBuffer(duplicate))
+                nodeIdCount = 1;
+                nodeIds[0] = cfk.redundantBefore().node.id;
+                for (int i = 0 ; i < commandCount ; ++i)
                 {
-                    depsIdsLocalSerializer.serialize(deps.txnIds(), out);
+                    if (nodeIdCount + 1 >= nodeIds.length)
+                    {
+                        nodeIdCount = compact(nodeIds);
+                        if (nodeIdCount > nodeIds.length/2)
+                            nodeIds = cachedInts().resize(nodeIds, nodeIds.length, nodeIds.length * 2);
+                    }
+
+                    TxnId txnId = cfk.txnId(i);
+                    Info info = cfk.info(i);
+
+                    hasNonStandardFlags |= txnIdFlags(txnId) != STANDARD;
+                    nodeIds[nodeIdCount++] = txnId.node.id;
+
+                    if (info.getClass() == NoInfo.class)
+                        continue;
+
+                    missingIdCount += info.missing.length;
+
+                    if (info.executeAt == txnId)
+                        continue;
+
+                    nodeIds[nodeIdCount++] = info.executeAt.node.id;
+                    bitsPerExecuteAtEpochDelta = Math.max(bitsPerExecuteAtEpochDelta, numberOfBitsToRepresent(info.executeAt.epoch() - txnId.epoch()));
+                    bitsPerExecuteAtHlcDelta = Math.max(bitsPerExecuteAtHlcDelta, numberOfBitsToRepresent(info.executeAt.hlc() - txnId.hlc()));
+                    bitsPerExecuteAtFlags = Math.max(bitsPerExecuteAtFlags, numberOfBitsToRepresent(info.executeAt.flags()));
+                    executeAtCount += 1;
                 }
-                catch (IOException e)
+                nodeIdCount = compact(nodeIds);
+                Invariants.checkState(nodeIdCount > 0);
+            }
+
+            // We can now use this information to calculate the fixed header size, compute the amount
+            // of additional space we'll need to store the TxnId and its basic info
+            int bitsPerNodeId = numberOfBitsToRepresent(nodeIdCount);
+            int minHeaderBits = 7 + bitsPerNodeId + (hasNonStandardFlags ? 1 : 0);
+            int infoHeaderBits = (executeAtCount > 0 ? 1 : 0) + (missingIdCount > 0 ? 1 : 0);
+            int maxHeaderBits = minHeaderBits;
+            int totalBytes = 0;
+
+            long prevEpoch = cfk.redundantBefore().epoch();
+            long prevHlc = cfk.redundantBefore().hlc();
+            int[] bytesHistogram = cachedInts().getInts(12);
+            Arrays.fill(bytesHistogram, 0);
+            for (int i = 0 ; i < commandCount ; ++i)
+            {
+                int headerBits = minHeaderBits;
+                int payloadBits = 0;
+
+                TxnId txnId = cfk.txnId(i);
                 {
-                    throw new RuntimeException(e);
+                    long epoch = txnId.epoch();
+                    Invariants.checkState(epoch >= prevEpoch);
+                    long epochDelta = epoch - prevEpoch;
+                    long hlc = txnId.hlc();
+                    long hlcDelta = hlc - prevHlc;
+
+                    if (epochDelta > 0)
+                    {
+                        if (hlcDelta < 0)
+                            hlcDelta = -1 - hlcDelta;
+
+                        headerBits += 3;
+                        if (epochDelta > 1)
+                        {
+                            if (epochDelta <= 0xf) headerBits += 4;
+                            else if (epochDelta <= 0xff) totalBytes += 1;
+                            else { totalBytes += 4; Invariants.checkState(epochDelta <= 0xffffffffL); }
+                        }
+                    }
+
+                    payloadBits += numberOfBitsToRepresent(hlcDelta);
+                    prevEpoch = epoch;
+                    prevHlc = hlc;
+                }
+
+                if (hasNonStandardFlags && txnIdFlags(txnId) == RAW)
+                    totalBytes += 2;
+
+                Info info = cfk.info(i);
+                if (info.status.hasInfo)
+                    headerBits += infoHeaderBits;
+                maxHeaderBits = Math.max(headerBits, maxHeaderBits);
+                int basicBytes = (headerBits + payloadBits + 7)/8;
+                bytesHistogram[basicBytes]++;
+            }
+
+            int minBasicBytes = -1, maxBasicBytes = 0;
+            for (int i = 0 ; i < bytesHistogram.length ; ++i)
+            {
+                if (bytesHistogram[i] == 0) continue;
+                if (minBasicBytes == -1) minBasicBytes = i;
+                maxBasicBytes = i;
+            }
+            for (int i = minBasicBytes + 1 ; i <= maxBasicBytes ; ++i)
+                bytesHistogram[i] += bytesHistogram[i-1];
+
+            int flags = (missingIdCount > 0 ? HAS_MISSING_DEPS_HEADER_BIT : 0)
+                        | (executeAtCount > 0 ? HAS_EXECUTE_AT_HEADER_BIT : 0)
+                        | (hasNonStandardFlags ? HAS_NON_STANDARD_FLAGS : 0);
+
+            int headerBytes = (maxHeaderBits+7)/8;
+            flags |= Invariants.checkArgument(headerBytes - 1, headerBytes <= 4) << 3;
+
+            int hlcBytesLookup;
+            {   // 2bits per size, first value may be zero and remainder may be increments of 1-4;
+                // only need to be able to encode a distribution of approx. 8 bytes at most, so
+                // pick lowest number we need first, then next lowest as 25th %ile while ensuring value of 1-4;
+                // then pick highest number we need, ensuring at least 2 greater than second (leaving room for third)
+                // then pick third number as 75th %ile, but at least 1 less than highest, and one more than second
+                // finally, ensure third then second are distributed so that there is no more than a gap of 4 between them and the next
+                int l0 = Math.max(0, Math.min(3, minBasicBytes - headerBytes));
+                int l1 = Math.max(l0+1, Math.min(l0+4,Arrays.binarySearch(bytesHistogram, commandCount/4) - headerBytes));
+                int l3 = Math.max(l1+2, maxBasicBytes - headerBytes);
+                int l2 = Math.max(l1+1, Math.min(l3-1, Arrays.binarySearch(bytesHistogram, (3*commandCount)/4) - headerBytes));
+                while (l3-l2 > 4) ++l2;
+                while (l2-l1 > 4) ++l1;
+                hlcBytesLookup = setHlcBytes(l0, l1, l2, l3);
+                flags |= (l0 | ((l1-(1+l0))<<2) | ((l2-(1+l1))<<4) | ((l3-(1+l2))<<6)) << 5;
+            }
+            int hlcFlagLookup = hlcBytesLookupToHlcFlagLookup(hlcBytesLookup);
+
+            totalBytes += bytesHistogram[minBasicBytes] * (headerBytes + getHlcBytes(hlcBytesLookup, getHlcFlag(hlcFlagLookup, minBasicBytes - headerBytes)));
+            for (int i = minBasicBytes + 1 ; i <= maxBasicBytes ; ++i)
+                totalBytes += (bytesHistogram[i] - bytesHistogram[i-1]) * (headerBytes + getHlcBytes(hlcBytesLookup, getHlcFlag(hlcFlagLookup, i - headerBytes)));
+            totalBytes += TypeSizes.sizeofUnsignedVInt(commandCount);
+            totalBytes += TypeSizes.sizeofUnsignedVInt(nodeIdCount);
+            totalBytes += TypeSizes.sizeofUnsignedVInt(nodeIds[0]);
+            for (int i = 1 ; i < nodeIdCount ; ++i)
+                totalBytes += TypeSizes.sizeofUnsignedVInt(nodeIds[i] - nodeIds[i-1]);
+            totalBytes += 2;
+
+            Arrays.fill(bytesHistogram, minBasicBytes, maxBasicBytes + 1, 0);
+            cachedInts().forceDiscard(bytesHistogram);
+
+            prevEpoch = cfk.redundantBefore().epoch();
+            prevHlc = cfk.redundantBefore().hlc();
+            // account for encoding redundantBefore
+            totalBytes += TypeSizes.sizeofUnsignedVInt(prevEpoch);
+            totalBytes += TypeSizes.sizeofUnsignedVInt(prevHlc);
+            totalBytes += 2; // flags TODO (expected): pack this along with uniqueIdBits, as usually zero bits should be needed
+            totalBytes += (bitsPerNodeId+7)/8;
+
+            if (missingIdCount + executeAtCount > 0)
+            {
+                // account for encoding missing id stream
+                int missingIdBits = 1 + numberOfBitsToRepresent(commandCount);
+                int executeAtBits = bitsPerNodeId
+                                    + bitsPerExecuteAtEpochDelta
+                                    + bitsPerExecuteAtHlcDelta
+                                    + bitsPerExecuteAtFlags;
+                totalBytes += (missingIdBits * missingIdCount + executeAtBits * executeAtCount + 7)/8;
+                if (executeAtCount > 0)
+                    totalBytes += 2;
+            }
+
+            ByteBuffer out = ByteBuffer.allocate(totalBytes);
+            VIntCoding.writeUnsignedVInt32(commandCount, out);
+            VIntCoding.writeUnsignedVInt32(nodeIdCount, out);
+            VIntCoding.writeUnsignedVInt32(nodeIds[0], out);
+            for (int i = 1 ; i < nodeIdCount ; ++i) // TODO (desired): can encode more efficiently as a stream of N bit integers
+                VIntCoding.writeUnsignedVInt32(nodeIds[i] - nodeIds[i-1], out);
+            out.putShort((short)flags);
+
+            VIntCoding.writeUnsignedVInt(prevEpoch, out);
+            VIntCoding.writeUnsignedVInt(prevHlc, out);
+            out.putShort((short) cfk.redundantBefore().flags());
+            writeLeastSignificantBytes(Arrays.binarySearch(nodeIds, 0, nodeIdCount, cfk.redundantBefore().node.id), (bitsPerNodeId+7)/8, out);
+
+            int executeAtMask = executeAtCount > 0 ? 1 : 0;
+            int missingDepsMask = missingIdCount > 0 ? 1 : 0;
+            int flagsIncrement = hasNonStandardFlags ? 2 : 1;
+            // TODO (desired): check this loop compiles correctly to only branch on epoch case, for binarySearch and flushing
+            for (int i = 0 ; i < commandCount ; ++i)
+            {
+                TxnId txnId = cfk.txnId(i);
+                Info info = cfk.info(i);
+                InternalStatus status = info.status;
+
+                long bits = status.ordinal();
+                int bitIndex = 3;
+
+                int statusHasInfo = status.hasInfo ? 1 : 0;
+                long hasExecuteAt = info.executeAt != null & info.executeAt != txnId ? 1 : 0;
+                bits |= hasExecuteAt << bitIndex;
+                bitIndex += statusHasInfo & executeAtMask;
+
+                long hasMissingIds = info.missing != CommandsForKey.NO_TXNIDS ? 1 : 0;
+                bits |= hasMissingIds << bitIndex;
+                bitIndex += statusHasInfo & missingDepsMask;
+
+                long flagBits = txnIdFlagsBits(txnId);
+                boolean writeFullFlags = flagBits == RAW_BITS;
+                bits |= flagBits << bitIndex;
+                bitIndex += flagsIncrement;
+
+                long hlcBits;
+                int extraEpochDeltaBytes = 0;
+                {
+                    long epoch = txnId.epoch();
+                    long delta = epoch - prevEpoch;
+                    long hlc = txnId.hlc();
+                    hlcBits = hlc - prevHlc;
+                    if (delta == 0)
+                    {
+                        bitIndex++;
+                    }
+                    else
+                    {
+                        bits |= 1L << bitIndex++;
+                        if (hlcBits < 0)
+                        {
+                            hlcBits = -1 - hlcBits;
+                            bits |= 1L << bitIndex;
+                        }
+                        bitIndex++;
+                        if (delta > 1)
+                        {
+                            if (delta <= 0xf)
+                            {
+                                bits |= 1L << bitIndex;
+                                bits |= delta << (bitIndex + 2);
+                                bitIndex += 4;
+                            }
+                            else
+                            {
+                                bits |= (delta <= 0xff ? 2L : 3L) << bitIndex;
+                                extraEpochDeltaBytes = Ints.checkedCast(delta);
+                            }
+                        }
+                        bitIndex += 2;
+                    }
+                    prevEpoch = epoch;
+                    prevHlc = hlc;
+                }
+
+                bits |= ((long)Arrays.binarySearch(nodeIds, 0, nodeIdCount, txnId.node.id)) << bitIndex;
+                bitIndex += bitsPerNodeId;
+
+                bits |= hlcBits << (bitIndex + 2);
+                hlcBits >>>= 8*headerBytes - (bitIndex + 2);
+                int hlcFlag = getHlcFlag(hlcFlagLookup, (7 + numberOfBitsToRepresent(hlcBits))/8);
+                bits |= ((long)hlcFlag) << bitIndex;
+
+                writeLeastSignificantBytes(bits, headerBytes, out);
+                writeLeastSignificantBytes(hlcBits, getHlcBytes(hlcBytesLookup, hlcFlag), out);
+
+                if (writeFullFlags)
+                    out.putShort((short)txnId.flags());
+
+                if (extraEpochDeltaBytes > 0)
+                {
+                    if (extraEpochDeltaBytes <= 0xff) out.put((byte)extraEpochDeltaBytes);
+                    else out.putInt(extraEpochDeltaBytes);
                 }
             }
-            return buffer;
-        }
 
-        @Override
-        public TxnId txnId(ByteBuffer data)
-        {
-            return CommandSerializers.txnId.deserialize(data, accessor, TXNID_OFFSET);
-        }
-
-        @Override
-        public Timestamp executeAt(ByteBuffer data)
-        {
-            byte flags = accessor.getByte(data, FLAG_OFFSET);
-            if ((flags & HAS_EXECUTE_AT) == 0)
-                return null;
-            return CommandSerializers.timestamp.deserialize(data, accessor, EXECUTEAT_OFFSET);
-        }
-
-        @Override
-        public SaveStatus saveStatus(ByteBuffer data)
-        {
-            return SaveStatus.values()[accessor.getByte(data, STATUS_OFFSET)];
-        }
-
-        @Override
-        public List<TxnId> depsIds(ByteBuffer data)
-        {
-            byte flags = accessor.getByte(data, FLAG_OFFSET);
-            if ((flags & HAS_DEPS) == 0)
-                return null;
-            ByteBuffer buffer = data.duplicate();
-            int offset = (flags & HAS_EXECUTE_AT) == 0 ? EXECUTEAT_OFFSET : DEPS_OFFSET;
-            buffer.position(data.position() + offset);
-            try (DataInputBuffer in = new DataInputBuffer(buffer, false))
+            if ((executeAtCount | missingIdCount) > 0)
             {
-                return depsIdsLocalSerializer.deserialize(in);
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
+                int bitsPerCommandId =  numberOfBitsToRepresent(commandCount);
+                int bitsPerMissingId = 1 + bitsPerCommandId;
+                int bitsPerExecuteAt = bitsPerExecuteAtEpochDelta + bitsPerExecuteAtHlcDelta + bitsPerExecuteAtFlags + bitsPerNodeId;
+                Invariants.checkState(bitsPerExecuteAtEpochDelta < 64);
+                Invariants.checkState(bitsPerExecuteAtHlcDelta <= 64);
+                Invariants.checkState(bitsPerExecuteAtFlags <= 16);
+                if (executeAtMask > 0) // we encode both 15 and 16 bits for flag length as 15 to fit in a short
+                    out.putShort((short) ((bitsPerExecuteAtEpochDelta << 10) | ((bitsPerExecuteAtHlcDelta-1) << 4) | (Math.min(15, bitsPerExecuteAtFlags))));
+                long buffer = 0L;
+                int bufferCount = 0;
+
+                for (int i = 0 ; i < commandCount ; ++i)
+                {
+                    Info info = cfk.info(i);
+                    if (info.getClass() == NoInfo.class)
+                        continue;
+
+                    TxnId txnId = cfk.txnId(i);
+                    if (info.executeAt != txnId)
+                    {
+                        Timestamp executeAt = info.executeAt;
+                        int nodeIdx = Arrays.binarySearch(nodeIds, 0, nodeIdCount, executeAt.node.id);
+                        if (bitsPerExecuteAt <= 64)
+                        {
+                            Invariants.checkState(executeAt.epoch() >= txnId.epoch());
+                            long executeAtBits = executeAt.epoch() - txnId.epoch();
+                            int offset = bitsPerExecuteAtEpochDelta;
+                            executeAtBits |= (executeAt.hlc() - txnId.hlc()) << offset ;
+                            offset += bitsPerExecuteAtHlcDelta;
+                            executeAtBits |= ((long)executeAt.flags()) << offset;
+                            offset += bitsPerExecuteAtFlags;
+                            executeAtBits |= ((long)nodeIdx) << offset;
+                            buffer = flushBits(buffer, bufferCount, executeAtBits, bitsPerExecuteAt, out);
+                            bufferCount = (bufferCount + bitsPerExecuteAt) & 63;
+                        }
+                        else
+                        {
+                            buffer = flushBits(buffer, bufferCount, executeAt.epoch() - txnId.epoch(), bitsPerExecuteAtEpochDelta, out);
+                            bufferCount = (bufferCount + bitsPerExecuteAtEpochDelta) & 63;
+                            buffer = flushBits(buffer, bufferCount, executeAt.hlc() - txnId.hlc(), bitsPerExecuteAtHlcDelta, out);
+                            bufferCount = (bufferCount + bitsPerExecuteAtHlcDelta) & 63;
+                            buffer = flushBits(buffer, bufferCount, executeAt.flags(), bitsPerExecuteAtFlags, out);
+                            bufferCount = (bufferCount + bitsPerExecuteAtFlags) & 63;
+                            buffer = flushBits(buffer, bufferCount, nodeIdx, bitsPerNodeId, out);
+                            bufferCount = (bufferCount + bitsPerNodeId) & 63;
+                        }
+                    }
+
+                    if (info.missing.length > 0)
+                    {
+                        int j = 0;
+                        while (j < info.missing.length - 1)
+                        {
+                            int missingId = cfk.indexOf(info.missing[j++]);
+                            buffer = flushBits(buffer, bufferCount, missingId, bitsPerMissingId, out);
+                            bufferCount = (bufferCount + bitsPerMissingId) & 63;
+                        }
+                        int missingId = cfk.indexOf(info.missing[info.missing.length - 1]);
+                        missingId |= 1L << bitsPerCommandId;
+                        buffer = flushBits(buffer, bufferCount, missingId, bitsPerMissingId, out);
+                        bufferCount = (bufferCount + bitsPerMissingId) & 63;
+                    }
+                }
+
+                writeMostSignificantBytes(buffer, (bufferCount + 7)/8, out);
             }
+
+            out.flip();
+            return out;
+        }
+        finally
+        {
+            cachedInts().forceDiscard(nodeIds);
         }
     }
+
+    private static long flushBits(long buffer, int bufferCount, long add, int addCount, ByteBuffer out)
+    {
+        Invariants.checkArgument(addCount == 64 || 0 == (add & (-1L << addCount)));
+        int total = bufferCount + addCount;
+        if (total < 64)
+        {
+            return buffer | (add << 64 - total);
+        }
+        else
+        {
+            buffer |= add >>> total - 64;
+            out.putLong(buffer);
+            return total == 64 ? 0 : (add << (128 - total));
+        }
+    }
+
+    public static CommandsForKey fromBytes(Key key, ByteBuffer in)
+    {
+        if (!in.hasRemaining())
+            return null;
+
+        in = in.duplicate();
+        int commandCount = VIntCoding.readUnsignedVInt32(in);
+        if (commandCount == 0)
+        {
+            long epoch = VIntCoding.readUnsignedVInt(in);
+            long hlc = VIntCoding.readUnsignedVInt(in);
+            int flags = VIntCoding.readUnsignedVInt32(in);
+            Node.Id id = new Node.Id(VIntCoding.readUnsignedVInt32(in));
+            return new CommandsForKey(key).withoutRedundant(TxnId.fromValues(epoch, hlc, flags, id));
+        }
+
+        TxnId[] txnIds = new TxnId[commandCount];
+        Info[] infos = new Info[commandCount];
+        int nodeIdCount = VIntCoding.readUnsignedVInt32(in);
+        int bitsPerNodeId = numberOfBitsToRepresent(nodeIdCount);
+        long nodeIdMask = (1L << bitsPerNodeId) - 1;
+        Node.Id[] nodeIds = new Node.Id[nodeIdCount]; // TODO (expected): use a shared reusable scratch buffer
+        {
+            int prev = VIntCoding.readUnsignedVInt32(in);
+            nodeIds[0] = new Node.Id(prev);
+            for (int i = 1 ; i < nodeIdCount ; ++i)
+                nodeIds[i] = new Node.Id(prev += VIntCoding.readUnsignedVInt32(in));
+        }
+
+        int missingDepsMasks, executeAtMasks, txnIdFlagsMask;
+        int headerByteCount, hlcBytesLookup;
+        {
+            int flags = in.getShort();
+            missingDepsMasks = 0 != (flags & HAS_MISSING_DEPS_HEADER_BIT) ? 1 : 0;
+            executeAtMasks = 0 != (flags & HAS_EXECUTE_AT_HEADER_BIT) ? 1 : 0;
+            txnIdFlagsMask = 0 != (flags & HAS_NON_STANDARD_FLAGS) ? 3 : 1;
+            headerByteCount = 1 + ((flags >>> 3) & 0x3);
+            hlcBytesLookup = setHlcByteDeltas((flags >>> 5) & 0x3, (flags >>> 7) & 0x3, (flags >>> 9) & 0x3, (flags >>> 11) & 0x3);
+        }
+
+        long prevEpoch = VIntCoding.readUnsignedVInt32(in);
+        long prevHlc = VIntCoding.readUnsignedVInt32(in);
+        TxnId redundantBefore = TxnId.fromValues(prevEpoch, prevHlc, in.getShort(),
+                                                 nodeIds[(int)readLeastSignificantBytes((bitsPerNodeId+7)/8, in)]);
+
+
+        for (int i = 0 ; i < commandCount ; ++i)
+        {
+            long header = readLeastSignificantBytes(headerByteCount, in);
+            header |= 1L << (8 * headerByteCount); // marker so we know where to shift-left most-significant bytes to
+            InternalStatus status = InternalStatus.get((int) (header & 0x7));
+            header >>>= 3;
+
+            int executeAtInfoOffset, missingDepsInfoOffset;
+            {
+                int infoMask = status.hasInfo ? 1 : 0;
+                int executeAtMask = infoMask & executeAtMasks, missingDepsMask = infoMask & missingDepsMasks;
+                executeAtInfoOffset = ((int)header & executeAtMask) << 1;
+                header >>>= executeAtMask;
+                missingDepsInfoOffset = (int)header & missingDepsMask;
+                header >>>= missingDepsMask;
+            }
+
+            Txn.Kind kind = TXN_ID_FLAG_BITS_KIND_LOOKUP[((int)header & txnIdFlagsMask)];
+            header >>>= Integer.bitCount(txnIdFlagsMask);
+
+            boolean hlcIsNegative = false;
+            long epoch = prevEpoch;
+            int readEpochBytes = 0;
+            {
+                boolean hasEpochDelta = (header & 1) == 1;
+                header >>>= 1;
+                if (hasEpochDelta)
+                {
+                    hlcIsNegative = (header & 1) == 1;
+                    header >>>= 1;
+
+                    int epochFlag = ((int)header & 0x3);
+                    header >>>= 2;
+                    switch (epochFlag)
+                    {
+                        default: throw new AssertionError("Unexpected value not 0-3");
+                        case 0: ++epoch; break;
+                        case 1: epoch += (header & 0xf); header >>>= 4; break;
+                        case 2: readEpochBytes = 1; break;
+                        case 3: readEpochBytes = 4; break;
+                    }
+                }
+            }
+
+            Node.Id node = nodeIds[(int)(header & nodeIdMask)];
+            header >>>= bitsPerNodeId;
+
+            int readHlcBytes = getHlcBytes(hlcBytesLookup, (int)(header & 0x3));
+            header >>>= 2;
+
+            long hlc = header;
+            {
+                long highestBit = Long.highestOneBit(hlc);
+                hlc ^= highestBit;
+                int hlcShift = Long.numberOfTrailingZeros(highestBit);
+                hlc |= readLeastSignificantBytes(readHlcBytes, in) << hlcShift;
+            }
+            if (hlcIsNegative)
+                hlc = -1-hlc;
+            hlc += prevHlc;
+
+            int flags = kind != null ? 0 : in.getShort();
+            if (readEpochBytes > 0)
+                epoch += readEpochBytes == 1 ? (in.get() & 0xff) : in.getInt();
+
+            TxnId txnId = kind != null ? new TxnId(epoch, hlc, kind, Domain.Key, node)
+                                       : TxnId.fromValues(epoch, hlc, flags, node);
+
+            txnIds[i] = txnId;
+            infos[i] = DECODE_INFOS[(executeAtInfoOffset | missingDepsInfoOffset)*STATUS_COUNT + status.ordinal()];
+
+            prevEpoch = epoch;
+            prevHlc = hlc;
+        }
+
+        if (executeAtMasks + missingDepsMasks > 0)
+        {
+            TxnId[] missingIdBuffer = cachedTxnIds().get(8);
+            int missingIdCount = 0, maxIdBufferCount = 0;
+            int bitsPerTxnId = numberOfBitsToRepresent(commandCount);
+            int txnIdMask = (1 << bitsPerTxnId) - 1;
+            int bitsPerMissingId = bitsPerTxnId + 1;
+
+            int decodeBits = executeAtMasks > 0 ? in.getShort() & 0xffff : 0;
+            int bitsPerEpochDelta = decodeBits >>> 10;
+            int bitsPerHlcDelta = 1 + ((decodeBits >>> 4) & 0x3f);
+            int bitsPerFlags = decodeBits & 0xf;
+            if (bitsPerFlags == 15) bitsPerFlags = 16;
+            int bitsPerExecuteAt = bitsPerEpochDelta + bitsPerHlcDelta + bitsPerFlags + bitsPerNodeId;
+
+            long epochDeltaMask = bitsPerEpochDelta == 0 ? 0 : (-1L >>> (64 - bitsPerEpochDelta));
+            long hlcDeltaMask = (-1L >>> (64 - bitsPerHlcDelta));
+            long flagsMask = bitsPerFlags == 0 ? 0 : (-1L >>> (64 - bitsPerFlags));
+
+            final BitReader reader = new BitReader();
+
+            for (int i = 0 ; i < commandCount ; ++i)
+            {
+                Info info = infos[i];
+                if (info.getClass() == NoInfo.class)
+                    continue;
+
+                TxnId txnId = txnIds[i];
+                Timestamp executeAt = txnId;
+                if (info.executeAt == null)
+                {
+                    long epoch, hlc;
+                    int flags;
+                    Node.Id id;
+                    if (bitsPerExecuteAt <= 64)
+                    {
+                        long executeAtBits = reader.read(bitsPerExecuteAt, in);
+                        epoch = txnId.epoch() + (executeAtBits & epochDeltaMask);
+                        executeAtBits >>>= bitsPerEpochDelta;
+                        hlc = txnId.hlc() + (executeAtBits & hlcDeltaMask);
+                        executeAtBits >>>= bitsPerHlcDelta;
+                        flags = (int)(executeAtBits & flagsMask);
+                        executeAtBits >>>= bitsPerFlags;
+                        id = nodeIds[(int)(executeAtBits & nodeIdMask)];
+                    }
+                    else
+                    {
+                        epoch = txnId.epoch() + reader.read(bitsPerEpochDelta, in);
+                        hlc = txnId.hlc() + reader.read(bitsPerHlcDelta, in);
+                        flags = (int) reader.read(bitsPerFlags, in);
+                        id = nodeIds[(int)(reader.read(bitsPerNodeId, in))];
+                    }
+                    executeAt = Timestamp.fromValues(epoch, hlc, flags, id);
+                }
+
+                TxnId[] missing = info.missing;
+                if (missing == null)
+                {
+                    int prev = -1;
+                    while (true)
+                    {
+                        if (missingIdCount == missingIdBuffer.length)
+                            missingIdBuffer = cachedTxnIds().resize(missingIdBuffer, missingIdCount, missingIdCount * 2);
+
+                        int next = (int) reader.read(bitsPerMissingId, in);
+                        Invariants.checkState(next > prev);
+                        missingIdBuffer[missingIdCount++] = txnIds[next & txnIdMask];
+                        if (next >= commandCount)
+                            break; // finished this array
+                        prev = next;
+                    }
+
+                    missing = Arrays.copyOf(missingIdBuffer, missingIdCount);
+                    maxIdBufferCount = missingIdCount;
+                    missingIdCount = 0;
+                }
+
+                infos[i] = Info.create(txnId, info.status, executeAt, missing);
+            }
+
+            cachedTxnIds().forceDiscard(missingIdBuffer, maxIdBufferCount);
+        }
+
+        return CommandsForKey.SerializerSupport.create(key, redundantBefore, txnIds, infos);
+    }
+
+    private static int getHlcBytes(int lookup, int index)
+    {
+        return (lookup >>> (index * 4)) & 0xf;
+    }
+
+    private static int setHlcBytes(int value1, int value2, int value3, int value4)
+    {
+        return value1 | (value2 << 4) | (value3 << 8) | (value4 << 12);
+    }
+
+    private static int setHlcByteDeltas(int value1, int value2, int value3, int value4)
+    {
+        value2 += 1 + value1;
+        value3 += 1 + value2;
+        value4 += 1 + value3;
+        return setHlcBytes(value1, value2, value3, value4);
+    }
+
+    private static int getHlcFlag(int flagsLookup, int bytes)
+    {
+        return (flagsLookup >>> (bytes * 2)) & 0x3;
+    }
+
+    private static int hlcBytesLookupToHlcFlagLookup(int bytesLookup)
+    {
+        int flagsLookup = 0;
+        int flagIndex = 0;
+        for (int bytesIndex = 0 ; bytesIndex < 4 ; bytesIndex++)
+        {
+            int flagLimit = getHlcBytes(bytesLookup, bytesIndex);
+            while (flagIndex <= flagLimit)
+                flagsLookup |= bytesIndex << (2 * flagIndex++);
+        }
+        return flagsLookup;
+    }
+
+    private static int compact(int[] buffer)
+    {
+        Arrays.sort(buffer);
+        int count = 0;
+        int j = 0;
+        while (j < buffer.length)
+        {
+            int prev;
+            buffer[count++] = prev = buffer[j];
+            while (++j < buffer.length && buffer[j] == prev) {}
+        }
+        return count;
+    }
+
+    private static int numberOfBitsToRepresent(long value)
+    {
+        return 64 - Long.numberOfLeadingZeros(value);
+    }
+
+    private static int numberOfBitsToRepresent(int value)
+    {
+        return 32 - Integer.numberOfLeadingZeros(value);
+    }
+
+    static final class BitReader
+    {
+        private long bitBuffer;
+        private int bitCount;
+
+        long read(int readCount, ByteBuffer in)
+        {
+            long result = bitBuffer >>> (64 - readCount);
+            int remaining = bitCount - readCount;
+            if (remaining >= 0)
+            {
+                bitBuffer <<= readCount;
+                bitCount = remaining;
+            }
+            else if (in.remaining() >= 8)
+            {
+                readCount -= bitCount;
+                bitBuffer = in.getLong();
+                bitCount = 64 - readCount;
+                result |= (bitBuffer >>> bitCount);
+                bitBuffer <<= readCount;
+            }
+            else
+            {
+                readCount -= bitCount;
+                while (readCount > 8)
+                {
+                    long next = in.get() & 0xff;
+                    readCount -= 8;
+                    result |= next << readCount;
+                }
+                long next = in.get() & 0xff;
+                bitCount = 8 - readCount;
+                result |= next >>> bitCount;
+                bitBuffer = next << (64 - bitCount);
+            }
+            return result;
+        }
+    }
+
+    enum TxnIdFlags
+    {
+        STANDARD, EXTENDED, RAW;
+        static final int EXTENDED_BITS = 0x2;
+        static final int RAW_BITS = 0x3;
+    }
+
+    private static TxnIdFlags txnIdFlags(TxnId txnId)
+    {
+        if (txnId.flags() > Timestamp.IDENTITY_FLAGS || txnId.domain() != Domain.Key)
+            return RAW;
+        switch (txnId.kind())
+        {
+            default: throw new AssertionError("Unhandled Kind: " + txnId.kind());
+            case Read:
+            case Write:
+                return STANDARD;
+            case ExclusiveSyncPoint:
+                return EXTENDED;
+            case SyncPoint:
+            case LocalOnly:
+            case EphemeralRead:
+                return RAW;
+        }
+    }
+
+    private static long  txnIdFlagsBits(TxnId txnId)
+    {
+        switch (txnIdFlags(txnId))
+        {
+            default: throw new AssertionError("Unhandled TxnIdFlag: " + txnIdFlags(txnId));
+            case RAW: return RAW_BITS;
+            case EXTENDED: return EXTENDED_BITS;
+            case STANDARD:
+                return txnId.kind() == Read ? 0 : 1;
+        }
+    }
+
+    private static final Txn.Kind[] TXN_ID_FLAG_BITS_KIND_LOOKUP = new Txn.Kind[] { Read, Write, ExclusiveSyncPoint, null };
+    private static final int STATUS_COUNT = InternalStatus.values().length;
+    private static final Info[] DECODE_INFOS = new Info[4 * STATUS_COUNT];
+    static
+    {
+        for (InternalStatus status : InternalStatus.values())
+        {
+            int ordinal = status.ordinal();
+            DECODE_INFOS[ordinal] = status.asNoInfo;
+            DECODE_INFOS[STATUS_COUNT+ordinal] = Info.createMock(status, Timestamp.NONE, null);
+            DECODE_INFOS[2*STATUS_COUNT+ordinal] = Info.createMock(status, null, CommandsForKey.NO_TXNIDS);
+            DECODE_INFOS[3*STATUS_COUNT+ordinal] = Info.createMock(status, null, null);
+        }
+    }
+
 }

src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java

@@ -28,6 +28,7 @@ import accord.primitives.FullRoute;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialRoute;
 import accord.primitives.PartialTxn;
+import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
@@ -60,29 +61,30 @@ public class CommitSerializers
             kind.serialize(msg.kind, out, version);
             CommandSerializers.ballot.serialize(msg.ballot, out, version);
             CommandSerializers.timestamp.serialize(msg.executeAt, out, version);
-            CommandSerializers.nullablePartialTxn.serialize(msg.partialTxn, out, version);
-            DepsSerializer.partialDeps.serialize(msg.partialDeps, out, version);
+            KeySerializers.seekables.serialize(msg.keys, out, version);
+            CommandSerializers.nullablePartialTxn.serialize(msg.keys, msg.partialTxn, out, version);
+            DepsSerializer.partialDeps.serialize(msg.keys, msg.partialDeps, out, version);
             serializeNullable(msg.route, out, version, KeySerializers.fullRoute);
             serializeNullable(msg.readData, out, version, read);
         }
 
         protected abstract C deserializeCommit(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Commit.Kind kind,
                                                Ballot ballot, Timestamp executeAt,
-                                               @Nullable PartialTxn partialTxn, PartialDeps partialDeps,
+                                               Seekables<?, ?> keys, @Nullable PartialTxn partialTxn, PartialDeps partialDeps,
                                                @Nullable FullRoute<?> fullRoute, @Nullable ReadData read);
 
         @Override
         public C deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch) throws IOException
         {
-            return deserializeCommit(txnId, scope, waitForEpoch,
-                                     kind.deserialize(in, version),
-                                     CommandSerializers.ballot.deserialize(in, version),
-                                     CommandSerializers.timestamp.deserialize(in, version),
-                                     CommandSerializers.nullablePartialTxn.deserialize(in, version),
-                                     DepsSerializer.partialDeps.deserialize(in, version),
-                                     deserializeNullable(in, version, KeySerializers.fullRoute),
-                                     deserializeNullable(in, version, read)
-            );
+            Commit.Kind kind = CommitSerializers.kind.deserialize(in, version);
+            Ballot ballot = CommandSerializers.ballot.deserialize(in, version);
+            Timestamp executeAt = CommandSerializers.timestamp.deserialize(in, version);
+            Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
+            PartialTxn txn = CommandSerializers.nullablePartialTxn.deserialize(keys, in, version);
+            PartialDeps deps = DepsSerializer.partialDeps.deserialize(keys, in, version);
+            FullRoute<?> route = deserializeNullable(in, version, KeySerializers.fullRoute);
+            ReadData read = deserializeNullable(in, version, this.read);
+            return deserializeCommit(txnId, scope, waitForEpoch, kind, ballot, executeAt, keys, txn, deps, route, read);
         }
 
         @Override
@@ -91,8 +93,9 @@ public class CommitSerializers
             return kind.serializedSize(msg.kind, version)
                    + CommandSerializers.ballot.serializedSize(msg.ballot, version)
                    + CommandSerializers.timestamp.serializedSize(msg.executeAt, version)
-                   + CommandSerializers.nullablePartialTxn.serializedSize(msg.partialTxn, version)
-                   + DepsSerializer.partialDeps.serializedSize(msg.partialDeps, version)
+                   + KeySerializers.seekables.serializedSize(msg.keys, version)
+                   + CommandSerializers.nullablePartialTxn.serializedSize(msg.keys, msg.partialTxn, version)
+                   + DepsSerializer.partialDeps.serializedSize(msg.keys, msg.partialDeps, version)
                    + serializedNullableSize(msg.route, version, KeySerializers.fullRoute)
                    + serializedNullableSize(msg.readData, version, read);
         }
@@ -101,13 +104,13 @@ public class CommitSerializers
     public static final IVersionedSerializer<Commit> request = new CommitSerializer<Commit, ReadData>(ReadData.class, ReadDataSerializers.readData)
     {
         @Override
-        protected Commit deserializeCommit(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Commit.Kind kind, Ballot ballot, Timestamp executeAt, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nullable ReadData read)
+        protected Commit deserializeCommit(TxnId txnId, PartialRoute<?> scope, long waitForEpoch, Commit.Kind kind, Ballot ballot, Timestamp executeAt, Seekables<?, ?> keys, @Nullable PartialTxn partialTxn, PartialDeps partialDeps, @Nullable FullRoute<?> fullRoute, @Nullable ReadData read)
         {
-            return Commit.SerializerSupport.create(txnId, scope, waitForEpoch, kind, ballot, executeAt, partialTxn, partialDeps, fullRoute, read);
+            return Commit.SerializerSupport.create(txnId, scope, waitForEpoch, kind, ballot, executeAt, keys, partialTxn, partialDeps, fullRoute, read);
         }
     };
 
-    public static final IVersionedSerializer<Commit.Invalidate> invalidate = new IVersionedSerializer<Commit.Invalidate>()
+    public static final IVersionedSerializer<Commit.Invalidate> invalidate = new IVersionedSerializer<>()
     {
         @Override
         public void serialize(Commit.Invalidate invalidate, DataOutputPlus out, int version) throws IOException

src/java/org/apache/cassandra/service/accord/serializers/DepsSerializer.java

@@ -28,6 +28,7 @@ import accord.primitives.PartialDeps;
 import accord.primitives.Range;
 import accord.primitives.RangeDeps;
 import accord.primitives.Ranges;
+import accord.primitives.Seekables;
 import accord.primitives.TxnId;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -39,9 +40,10 @@ import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIds;
 import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIdsCount;
 import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIds;
 import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIdsCount;
+import static accord.primitives.Routable.Domain.Key;
 import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
 
-public abstract class DepsSerializer<D extends Deps> implements IVersionedSerializer<D>
+public abstract class DepsSerializer<D extends Deps> extends IVersionedWithKeysSerializer.AbstractWithKeysSerializer implements IVersionedWithKeysSerializer<Seekables<?, ?>, D>
 {
     public static final DepsSerializer<Deps> deps = new DepsSerializer<Deps>()
     {
@@ -69,23 +71,85 @@ public abstract class DepsSerializer<D extends Deps> implements IVersionedSerial
             KeySerializers.ranges.serialize(partialDeps.covering, out, version);
         }
 
+        @Override
+        public void serialize(Seekables<?, ?> keys, PartialDeps partialDeps, DataOutputPlus out, int version) throws IOException
+        {
+            super.serialize(keys, partialDeps, out, version);
+            KeySerializers.ranges.serialize(partialDeps.covering, out, version);
+        }
+
         @Override
         public long serializedSize(PartialDeps partialDeps, int version)
         {
             return super.serializedSize(partialDeps, version)
                  + KeySerializers.ranges.serializedSize(partialDeps.covering, version);
         }
+
+        @Override
+        public long serializedSize(Seekables<?, ?> keys, PartialDeps partialDeps, int version)
+        {
+            return super.serializedSize(keys, partialDeps, version)
+                 + KeySerializers.ranges.serializedSize(partialDeps.covering, version);
+        }
     };
 
     public static final IVersionedSerializer<PartialDeps> nullablePartialDeps = NullableSerializer.wrap(partialDeps);
 
+    abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException;
+
     @Override
     public void serialize(D deps, DataOutputPlus out, int version) throws IOException
+    {
+        KeySerializers.keys.serialize(deps.keyDeps.keys(), out, version);
+        serializeWithoutKeys(deps, out, version);
+    }
+
+    @Override
+    public void serialize(Seekables<?, ?> keys, D deps, DataOutputPlus out, int version) throws IOException
+    {
+        if (keys.domain() == Key) serializeSubset(deps.keyDeps.keys(), keys, out);
+        else KeySerializers.keys.serialize(deps.keyDeps.keys(), out, version);
+        serializeWithoutKeys(deps, out, version);
+    }
+
+    @Override
+    public D deserialize(DataInputPlus in, int version) throws IOException
+    {
+        Keys keys = KeySerializers.keys.deserialize(in, version);
+        return deserializeWithoutKeys(keys, in, version);
+    }
+
+    @Override
+    public D deserialize(Seekables<?, ?> superset, DataInputPlus in, int version) throws IOException
+    {
+        Keys keys;
+        if (superset.domain() == Key) keys = (Keys)deserializeSubset(superset, in);
+        else keys = KeySerializers.keys.deserialize(in, version);
+        return deserializeWithoutKeys(keys, in, version);
+    }
+
+    @Override
+    public long serializedSize(D deps, int version)
+    {
+        long size = KeySerializers.keys.serializedSize(deps.keyDeps.keys(), version);
+        size += serializedSizeWithoutKeys(deps, version);
+        return size;
+    }
+
+    @Override
+    public long serializedSize(Seekables<?, ?> keys, D deps, int version)
+    {
+        long size;
+        if (keys.domain() == Key) size = serializedSubsetSize(deps.keyDeps.keys(), keys);
+        else size = KeySerializers.keys.serializedSize(deps.keyDeps.keys(), version);
+        size += serializedSizeWithoutKeys(deps, version);
+        return size;
+    }
+
+    private void serializeWithoutKeys(D deps, DataOutputPlus out, int version) throws IOException
     {
         KeyDeps keyDeps = deps.keyDeps;
         {
-            KeySerializers.keys.serialize(keyDeps.keys(), out, version);
-
             int txnIdCount = keyDeps.txnIdCount();
             out.writeUnsignedVInt32(txnIdCount);
             for (int i = 0; i < txnIdCount; i++)
@@ -116,13 +180,10 @@ public abstract class DepsSerializer<D extends Deps> implements IVersionedSerial
         }
     }
 
-    @Override
-    public D deserialize(DataInputPlus in, int version) throws IOException
+    private D deserializeWithoutKeys(Keys keys, DataInputPlus in, int version) throws IOException
     {
         KeyDeps keyDeps;
         {
-            Keys keys = KeySerializers.keys.deserialize(in, version);
-
             int txnIdCount = in.readUnsignedVInt32();
             TxnId[] txnIds = new TxnId[txnIdCount];
             for (int i = 0; i < txnIdCount; i++)
@@ -159,17 +220,12 @@ public abstract class DepsSerializer<D extends Deps> implements IVersionedSerial
         return deserialize(keyDeps, rangeDeps, in, version);
     }
 
-    abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, DataInputPlus in, int version) throws IOException;
-
-    @Override
-    public long serializedSize(D deps, int version)
+    private long serializedSizeWithoutKeys(D deps, int version)
     {
         long size = 0L;
 
         KeyDeps keyDeps = deps.keyDeps;
         {
-            size += KeySerializers.keys.serializedSize(keyDeps.keys(), version);
-
             int txnIdCount = keyDeps.txnIdCount();
             size += sizeofUnsignedVInt(txnIdCount);
             for (int i = 0; i < txnIdCount; i++)
@@ -201,4 +257,5 @@ public abstract class DepsSerializer<D extends Deps> implements IVersionedSerial
 
         return size;
     }
+
 }

src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java

@@ -30,7 +30,7 @@ import accord.local.Status.Durability;
 import accord.local.Status.Known;
 import accord.messages.CheckStatus;
 import accord.messages.Propagate;
-import accord.messages.ReadData;
+import accord.messages.ReadData.CommitOrReadNack;
 import accord.messages.ReadData.ReadReply;
 import accord.primitives.Ballot;
 import accord.primitives.PartialDeps;
@@ -40,7 +40,6 @@ import accord.primitives.Route;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.primitives.Writes;
-import accord.utils.Invariants;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -60,14 +59,11 @@ public class FetchSerializers
         @Override
         public void serialize(FetchRequest request, DataOutputPlus out, int version) throws IOException
         {
-            Invariants.checkArgument(request.txnId.epoch() == request.executeAt.epoch());
-
-            out.writeUnsignedVInt(request.waitForEpoch());
+            out.writeUnsignedVInt(request.executeAtEpoch);
             CommandSerializers.txnId.serialize(request.txnId, out, version);
             KeySerializers.ranges.serialize((Ranges) request.readScope, out, version);
             DepsSerializer.partialDeps.serialize(request.partialDeps, out, version);
             StreamingTxn.serializer.serialize(request.read, out, version);
-            out.writeBoolean(request.collectMaxApplied);
         }
 
         @Override
@@ -77,25 +73,23 @@ public class FetchSerializers
                                     CommandSerializers.txnId.deserialize(in, version),
                                     KeySerializers.ranges.deserialize(in, version),
                                     DepsSerializer.partialDeps.deserialize(in, version),
-                                    StreamingTxn.serializer.deserialize(in, version),
-                                    in.readBoolean());
+                                    StreamingTxn.serializer.deserialize(in, version));
         }
 
         @Override
         public long serializedSize(FetchRequest request, int version)
         {
-            return TypeSizes.sizeofUnsignedVInt(request.waitForEpoch())
+            return TypeSizes.sizeofUnsignedVInt(request.executeAtEpoch)
                    + CommandSerializers.txnId.serializedSize(request.txnId, version)
                    + KeySerializers.ranges.serializedSize((Ranges) request.readScope, version)
                    + DepsSerializer.partialDeps.serializedSize(request.partialDeps, version)
-                   + StreamingTxn.serializer.serializedSize(request.read, version)
-                   + TypeSizes.BYTE_SIZE;
+                   + StreamingTxn.serializer.serializedSize(request.read, version);
         }
     };
 
     public static final IVersionedSerializer<ReadReply> reply = new IVersionedSerializer<ReadReply>()
     {
-        final ReadData.CommitOrReadNack[] nacks = ReadData.CommitOrReadNack.values();
+        final CommitOrReadNack[] nacks = CommitOrReadNack.values();
         final IVersionedSerializer<Data> streamDataSerializer = new CastingSerializer<>(StreamData.class, StreamData.serializer);
 
         @Override
@@ -103,7 +97,7 @@ public class FetchSerializers
         {
             if (!reply.isOk())
             {
-                out.writeByte(1 + ((ReadData.CommitOrReadNack) reply).ordinal());
+                out.writeByte(1 + ((CommitOrReadNack) reply).ordinal());
                 return;
             }
 

src/java/org/apache/cassandra/service/accord/serializers/GetEphmrlReadDepsSerializers.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+
+import accord.messages.GetEphemeralReadDeps;
+import accord.messages.GetEphemeralReadDeps.GetEphemeralReadDepsOk;
+import accord.primitives.PartialDeps;
+import accord.primitives.PartialRoute;
+import accord.primitives.Seekables;
+import accord.primitives.TxnId;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class GetEphmrlReadDepsSerializers
+{
+    public static final IVersionedSerializer<GetEphemeralReadDeps> request = new TxnRequestSerializer.WithUnsyncedSerializer<GetEphemeralReadDeps>()
+    {
+        @Override
+        public void serializeBody(GetEphemeralReadDeps msg, DataOutputPlus out, int version) throws IOException
+        {
+            KeySerializers.seekables.serialize(msg.keys, out, version);
+            out.writeUnsignedVInt(msg.executionEpoch);
+        }
+
+        @Override
+        public GetEphemeralReadDeps deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch, long minEpoch, boolean doNotComputeProgressKey) throws IOException
+        {
+            Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
+            long executionEpoch = in.readUnsignedVInt();
+            return GetEphemeralReadDeps.SerializationSupport.create(txnId, scope, waitForEpoch, minEpoch, keys, executionEpoch);
+        }
+
+        @Override
+        public long serializedBodySize(GetEphemeralReadDeps msg, int version)
+        {
+            return KeySerializers.seekables.serializedSize(msg.keys, version)
+                   + TypeSizes.sizeofUnsignedVInt(msg.executionEpoch);
+        }
+    };
+
+    public static final IVersionedSerializer<GetEphemeralReadDepsOk> reply = new IVersionedSerializer<GetEphemeralReadDepsOk>()
+    {
+        @Override
+        public void serialize(GetEphemeralReadDepsOk reply, DataOutputPlus out, int version) throws IOException
+        {
+            DepsSerializer.partialDeps.serialize(reply.deps, out, version);
+            out.writeUnsignedVInt(reply.latestEpoch);
+        }
+
+        @Override
+        public GetEphemeralReadDepsOk deserialize(DataInputPlus in, int version) throws IOException
+        {
+            PartialDeps deps = DepsSerializer.partialDeps.deserialize(in, version);
+            long latestEpoch = in.readUnsignedVInt();
+            return new GetEphemeralReadDepsOk(deps, latestEpoch);
+        }
+
+        @Override
+        public long serializedSize(GetEphemeralReadDepsOk reply, int version)
+        {
+            return DepsSerializer.partialDeps.serializedSize(reply.deps, version)
+                   + TypeSizes.sizeofUnsignedVInt(reply.latestEpoch);
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/serializers/GetMaxConflictSerializers.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+
+import accord.messages.GetMaxConflict;
+import accord.messages.GetMaxConflict.GetMaxConflictOk;
+import accord.primitives.PartialRoute;
+import accord.primitives.Seekables;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+public class GetMaxConflictSerializers
+{
+    public static final IVersionedSerializer<GetMaxConflict> request = new TxnRequestSerializer.WithUnsyncedSerializer<GetMaxConflict>()
+    {
+        @Override
+        public void serializeBody(GetMaxConflict msg, DataOutputPlus out, int version) throws IOException
+        {
+            KeySerializers.seekables.serialize(msg.keys, out, version);
+            out.writeUnsignedVInt(msg.executionEpoch);
+        }
+
+        @Override
+        public GetMaxConflict deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch, long minEpoch, boolean doNotComputeProgressKey) throws IOException
+        {
+            Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
+            long executionEpoch = in.readUnsignedVInt();
+            return GetMaxConflict.SerializationSupport.create(scope, waitForEpoch, minEpoch, keys, executionEpoch);
+        }
+
+        @Override
+        public long serializedBodySize(GetMaxConflict msg, int version)
+        {
+            return KeySerializers.seekables.serializedSize(msg.keys, version)
+                   + TypeSizes.sizeofUnsignedVInt(msg.executionEpoch);
+        }
+    };
+
+    public static final IVersionedSerializer<GetMaxConflictOk> reply = new IVersionedSerializer<GetMaxConflictOk>()
+    {
+        @Override
+        public void serialize(GetMaxConflictOk reply, DataOutputPlus out, int version) throws IOException
+        {
+            CommandSerializers.timestamp.serialize(reply.maxConflict, out, version);
+            out.writeUnsignedVInt(reply.latestEpoch);
+        }
+
+        @Override
+        public GetMaxConflictOk deserialize(DataInputPlus in, int version) throws IOException
+        {
+            Timestamp maxConflict = CommandSerializers.timestamp.deserialize(in, version);
+            long latestEpoch = in.readUnsignedVInt();
+            return new GetMaxConflictOk(maxConflict, latestEpoch);
+        }
+
+        @Override
+        public long serializedSize(GetMaxConflictOk reply, int version)
+        {
+            return CommandSerializers.timestamp.serializedSize(reply.maxConflict, version)
+                   + TypeSizes.sizeofUnsignedVInt(reply.latestEpoch);
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/serializers/IVersionedWithKeysSerializer.java

@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+
+import accord.api.Key;
+import accord.primitives.AbstractKeys;
+import accord.primitives.Keys;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.RoutableKey;
+import accord.primitives.Routables;
+import accord.primitives.Seekables;
+import net.nicoulaj.compilecommand.annotations.DontInline;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+import static accord.utils.SortedArrays.Search.FAST;
+
+/**
+ * De/serialize a structure that can refer to a known superset of RoutingKeys/Keys/Ranges...
+ */
+public interface IVersionedWithKeysSerializer<K extends Routables<?>, T> extends IVersionedSerializer<T>
+{
+    /**
+     * Serialize the specified type into the specified DataOutputStream instance.
+     *
+     * @param t type that needs to be serialized
+     * @param out DataOutput into which serialization needs to happen.
+     * @param version protocol version
+     * @throws IOException if serialization fails
+     */
+    void serialize(K keys, T t, DataOutputPlus out, int version) throws IOException;
+
+    /**
+     * Deserialize into the specified DataInputStream instance.
+     * @param in DataInput from which deserialization needs to happen.
+     * @param version protocol version
+     * @return the type that was deserialized
+     * @throws IOException if deserialization fails
+     */
+    T deserialize(K keys, DataInputPlus in, int version) throws IOException;
+
+    /**
+     * Calculate serialized size of object without actually serializing.
+     * @param t object to calculate serialized size
+     * @param version protocol version
+     * @return serialized size of object t
+     */
+    long serializedSize(K keys, T t, int version);
+
+    final class NullableWithKeysSerializer<K extends Routables<?>, T> implements IVersionedWithKeysSerializer<K, T>
+    {
+        final IVersionedWithKeysSerializer<K, T> wrapped;
+        public NullableWithKeysSerializer(IVersionedWithKeysSerializer<K, T> wrapped)
+        {
+            this.wrapped = wrapped;
+        }
+
+        @Override
+        public void serialize(T t, DataOutputPlus out, int version) throws IOException
+        {
+            out.writeByte(t == null ? 0 : 1);
+            if (t != null) wrapped.serialize(t, out, version);
+        }
+
+        @Override
+        public T deserialize(DataInputPlus in, int version) throws IOException
+        {
+            if (in.readByte() == 0) return null;
+            return wrapped.deserialize(in, version);
+        }
+
+        @Override
+        public long serializedSize(T t, int version)
+        {
+            return t == null ? 1 : 1 + wrapped.serializedSize(t, version);
+        }
+
+        @Override
+        public void serialize(K keys, T t, DataOutputPlus out, int version) throws IOException
+        {
+            out.writeByte(t == null ? 0 : 1);
+            if (t != null) wrapped.serialize(keys, t, out, version);
+        }
+
+        @Override
+        public T deserialize(K keys, DataInputPlus in, int version) throws IOException
+        {
+            if (in.readByte() == 0) return null;
+            return wrapped.deserialize(keys, in, version);
+        }
+
+        @Override
+        public long serializedSize(K keys, T t, int version)
+        {
+            return t == null ? 1 : 1 + wrapped.serializedSize(keys, t, version);
+        }
+    }
+
+    abstract class AbstractWithKeysSerializer
+    {
+        /**
+         * If both ends have a pre-shared superset of the columns we are serializing, we can send them much
+         * more efficiently. Both ends must provide the identically same set of columns.
+         */
+        protected void serializeSubset(Seekables<?, ?> serialize, Seekables<?, ?> superset, DataOutputPlus out) throws IOException
+        {
+            /**
+             * We weight this towards small sets, and sets where the majority of items are present, since
+             * we expect this to mostly be used for serializing result sets.
+             *
+             * For supersets with fewer than 64 columns, we encode a bitmap of *missing* columns,
+             * which equates to a zero (single byte) when all columns are present, and otherwise
+             * a positive integer that can typically be vint encoded efficiently.
+             *
+             * If we have 64 or more columns, we cannot neatly perform a bitmap encoding, so we just switch
+             * to a vint encoded set of deltas, either adding or subtracting (whichever is most efficient).
+             * We indicate this switch by sending our bitmap with every bit set, i.e. -1L
+             */
+            int serializeCount = serialize.size();
+            int supersetCount = superset.size();
+            if (serializeCount == supersetCount)
+            {
+                out.writeUnsignedVInt(0L);
+            }
+            else if (supersetCount < 64)
+            {
+                switch (serialize.domain())
+                {
+                    default: throw new AssertionError("Unhandled domain: " + serialize.domain());
+                    case Key:
+                        out.writeUnsignedVInt(encodeBitmap((Keys)serialize, (Keys)superset, supersetCount));
+                        break;
+                    case Range:
+                        out.writeUnsignedVInt(encodeBitmap((Ranges)serialize, (Ranges)superset, supersetCount));
+                        break;
+                }
+            }
+            else
+            {
+                switch (serialize.domain())
+                {
+                    default: throw new AssertionError("Unhandled domain: " + serialize.domain());
+                    case Key:
+                        serializeLargeSubset((Keys)serialize, serializeCount, (Keys)superset, supersetCount, out);
+                        break;
+                    case Range:
+                        serializeLargeSubset((Ranges)serialize, serializeCount, (Ranges)superset, supersetCount, out);
+                        break;
+                }
+            }
+        }
+
+        public long serializedSubsetSize(Seekables<?, ?> serialize, Seekables<?, ?> superset)
+        {
+            int columnCount = serialize.size();
+            int supersetCount = superset.size();
+            if (columnCount == supersetCount)
+            {
+                return TypeSizes.sizeofUnsignedVInt(0);
+            }
+            else if (supersetCount < 64)
+            {
+                switch (serialize.domain())
+                {
+                    default: throw new AssertionError("Unhandled domain: " + serialize.domain());
+                    case Key:
+                        return TypeSizes.sizeofUnsignedVInt(encodeBitmap((Keys)serialize, (Keys)superset, supersetCount));
+                    case Range:
+                        return TypeSizes.sizeofUnsignedVInt(encodeBitmap((Ranges)serialize, (Ranges)superset, supersetCount));
+                }
+            }
+            else
+            {
+                switch (serialize.domain())
+                {
+                    default: throw new AssertionError("Unhandled domain: " + serialize.domain());
+                    case Key:
+                        return serializeLargeSubsetSize((Keys)serialize, columnCount, (Keys)superset, supersetCount);
+                    case Range:
+                        return serializeLargeSubsetSize((Ranges)serialize, columnCount, (Ranges)superset, supersetCount);
+                }
+            }
+        }
+
+        public Seekables<?, ?> deserializeSubset(Seekables<?, ?> superset, DataInputPlus in) throws IOException
+        {
+            long encoded = in.readUnsignedVInt();
+            int supersetCount = superset.size();
+            if (encoded == 0L)
+            {
+                return superset;
+            }
+            else if (supersetCount >= 64)
+            {
+                return deserializeLargeSubset(in, superset, supersetCount, (int) encoded);
+            }
+            else
+            {
+                encoded ^= -1L >>> (64 - supersetCount);
+                int deserializeCount = Long.bitCount(encoded);
+                switch (superset.domain())
+                {
+                    default: throw new AssertionError("Unhandled domain: " + superset.domain());
+                    case Key:
+                    {
+                        Keys keys = (Keys)superset;
+                        Key[] out = new Key[deserializeCount];
+                        int count = 0;
+                        while (encoded != 0)
+                        {
+                            long lowestBit = Long.lowestOneBit(encoded);
+                            out[count++] = keys.get(Long.numberOfTrailingZeros(lowestBit));
+                            encoded ^= lowestBit;
+                        }
+                        return Keys.ofSortedUnique(out);
+                    }
+                    case Range:
+                    {
+                        Ranges ranges = (Ranges)superset;
+                        Range[] out = new Range[deserializeCount];
+                        int count = 0;
+                        while (encoded != 0)
+                        {
+                            long lowestBit = Long.lowestOneBit(encoded);
+                            out[count++] = ranges.get(Long.numberOfTrailingZeros(lowestBit));
+                            encoded ^= lowestBit;
+                        }
+                        return Ranges.ofSortedAndDeoverlapped(out);
+                    }
+                }
+            }
+        }
+
+        // encodes a 1 bit for every *missing* column, on the assumption presence is more common,
+        // and because this is consistent with encoding 0 to represent all present
+        private static <K extends RoutableKey> long encodeBitmap(AbstractKeys<K> serialize, AbstractKeys<K> superset, int supersetCount)
+        {
+            // the index we would encounter next if all columns are present
+            long bitmap = superset.foldl(serialize, (k, p1, v, i) -> {
+                return v | (1L << i);
+            }, 0L, 0L, -1L);
+            bitmap ^= -1L >>> (64 - supersetCount);
+            return bitmap;
+        }
+
+        private static long encodeBitmap(Ranges serialize, Ranges superset, int supersetCount)
+        {
+            // the index we would encounter next if all columns are present
+            long bitmap = superset.foldl(serialize, (k, p1, v, i) -> {
+                return v | (1L << i);
+            }, 0L, 0L, -1L);
+            bitmap ^= -1L >>> (64 - supersetCount);
+            return bitmap;
+        }
+
+        @DontInline
+        private <K extends RoutableKey> void serializeLargeSubset(AbstractKeys<K> serialize, int serializeCount, AbstractKeys<K> superset, int supersetCount, DataOutputPlus out) throws IOException
+        {
+            out.writeUnsignedVInt32(supersetCount - serializeCount);
+            int serializeIndex = 0, supersetIndex = 0;
+            while (serializeIndex < serializeCount)
+            {
+                int prevSupersetIndex = supersetIndex;
+                int nextSupersetIndex;
+                do
+                {
+                    nextSupersetIndex = superset.findNext(supersetIndex, serialize.get(serializeIndex++), FAST);
+                    if (supersetIndex + 1 != nextSupersetIndex)
+                        break;
+                    supersetIndex++;
+                }
+                while (serializeIndex < serializeCount);
+
+                out.writeUnsignedVInt32(supersetIndex - prevSupersetIndex);
+                out.writeUnsignedVInt32(nextSupersetIndex - supersetIndex);
+                supersetIndex = nextSupersetIndex;
+            }
+        }
+
+        @DontInline
+        private void serializeLargeSubset(Ranges serialize, int serializeCount, Ranges superset, int supersetCount, DataOutputPlus out) throws IOException
+        {
+            out.writeUnsignedVInt32(supersetCount - serializeCount);
+            int serializeIndex = 0, supersetIndex = 0;
+            while (serializeIndex < serializeCount)
+            {
+                int prevSupersetIndex = supersetIndex;
+                int nextSupersetIndex;
+                do
+                {
+                    nextSupersetIndex = superset.findNext(supersetIndex, serialize.get(serializeIndex++), FAST);
+                    if (supersetIndex + 1 != nextSupersetIndex)
+                        break;
+                    supersetIndex++;
+                }
+                while (serializeIndex < serializeCount);
+
+                out.writeUnsignedVInt32(supersetIndex - prevSupersetIndex);
+                out.writeUnsignedVInt32(nextSupersetIndex - supersetIndex);
+                supersetIndex = nextSupersetIndex;
+            }
+        }
+
+        @DontInline
+        private Seekables<?, ?> deserializeLargeSubset(DataInputPlus in, Seekables<?, ?> superset, int supersetCount, int delta) throws IOException
+        {
+            int deserializeCount = supersetCount - delta;
+            switch (superset.domain())
+            {
+                default: throw new AssertionError("Unhandled domain: " + superset.domain());
+                case Key:
+                {
+                    Keys keys = (Keys)superset;
+                    Key[] out = new Key[deserializeCount];
+                    int supersetIndex = 0;
+                    int count = 0;
+                    while (count < deserializeCount)
+                    {
+                        int takeCount = in.readUnsignedVInt32();
+                        while (takeCount-- > 0) out[count++] = keys.get(supersetIndex++);
+                        supersetIndex += in.readUnsignedVInt32();
+                    }
+                    return Keys.ofSortedUnique(out);
+                }
+                case Range:
+                {
+                    Ranges ranges = (Ranges)superset;
+                    Range[] out = new Range[deserializeCount];
+                    int supersetIndex = 0;
+                    int count = 0;
+                    while (count < deserializeCount)
+                    {
+                        int takeCount = in.readUnsignedVInt32();
+                        while (takeCount-- > 0) out[count++] = ranges.get(supersetIndex++);
+                        supersetIndex += in.readUnsignedVInt32();
+                    }
+                    return Ranges.ofSortedAndDeoverlapped(out);
+                }
+            }
+        }
+
+        @DontInline
+        private <K extends RoutableKey> long serializeLargeSubsetSize(AbstractKeys<K> serialize, int serializeCount, AbstractKeys<K> superset, int supersetCount)
+        {
+            long size = TypeSizes.sizeofUnsignedVInt(supersetCount - serializeCount);
+            int serializeIndex = 0, supersetIndex = 0;
+            while (serializeIndex < serializeCount)
+            {
+                int prevSupersetIndex = supersetIndex;
+                int nextSupersetIndex;
+                do
+                {
+                    nextSupersetIndex = superset.findNext(supersetIndex, serialize.get(serializeIndex++), FAST);
+                    if (supersetIndex + 1 != nextSupersetIndex)
+                        break;
+                    supersetIndex++;
+                }
+                while (serializeIndex < serializeCount);
+
+                size += TypeSizes.sizeofUnsignedVInt(supersetIndex - prevSupersetIndex);
+                size += TypeSizes.sizeofUnsignedVInt(nextSupersetIndex - supersetIndex);
+                supersetIndex = nextSupersetIndex;
+            }
+            return size;
+        }
+
+        @DontInline
+        private long serializeLargeSubsetSize(Ranges serialize, int serializeCount, Ranges superset, int supersetCount)
+        {
+            long size = TypeSizes.sizeofUnsignedVInt(supersetCount - serializeCount);
+            int serializeIndex = 0, supersetIndex = 0;
+            while (serializeIndex < serializeCount)
+            {
+                int prevSupersetIndex = supersetIndex;
+                int nextSupersetIndex;
+                do
+                {
+                    nextSupersetIndex = superset.findNext(supersetIndex, serialize.get(serializeIndex++), FAST);
+                    if (supersetIndex + 1 != nextSupersetIndex)
+                        break;
+                    supersetIndex++;
+                }
+                while (serializeIndex < serializeCount);
+
+                size += TypeSizes.sizeofUnsignedVInt(supersetIndex - prevSupersetIndex);
+                size += TypeSizes.sizeofUnsignedVInt(nextSupersetIndex - supersetIndex);
+                supersetIndex = nextSupersetIndex;
+            }
+            return size;
+        }
+    }
+
+}

src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java

@@ -211,6 +211,8 @@ public class KeySerializers
         EnumSet.of(UnseekablesKind.FullKeyRoute, UnseekablesKind.FullRangeRoute)
     );
 
+    public static final IVersionedSerializer<FullRoute<?>> nullableFullRoute = NullableSerializer.wrap(fullRoute);
+
     public static final IVersionedSerializer<Unseekables<?>> unseekables = new AbstractRoutablesSerializer<>(
         EnumSet.allOf(UnseekablesKind.class)
     );
@@ -352,6 +354,8 @@ public class KeySerializers
         }
     };
 
+    public static final IVersionedSerializer<Seekables<?, ?>> nullableSeekables = NullableSerializer.wrap(seekables);
+
     public static abstract class AbstractKeysSerializer<K extends RoutableKey, KS extends AbstractKeys<K>> implements IVersionedSerializer<KS>
     {
         final IVersionedSerializer<K> keySerializer;

src/java/org/apache/cassandra/service/accord/serializers/ListenerSerializers.java

@@ -20,32 +20,23 @@ package org.apache.cassandra.service.accord.serializers;
 
 import java.io.IOException;
 
-import accord.impl.CommandsForKey;
 import accord.local.Command;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.CommandsForRanges;
-import org.apache.cassandra.service.accord.api.PartitionKey;
 
 public class ListenerSerializers
 {
     public enum Kind
     {
-        COMMAND, COMMANDS_FOR_KEY, COMMANDS_FOR_RANGE;
+        COMMAND;
 
         private static Kind of(Command.DurableAndIdempotentListener listener)
         {
             if (listener instanceof Command.ProxyListener)
                 return COMMAND;
 
-            if (listener instanceof CommandsForKey.Listener)
-                return COMMANDS_FOR_KEY;
-
-            if (listener instanceof CommandsForRanges.Listener)
-                return COMMANDS_FOR_RANGE;
-
             throw new IllegalArgumentException("Unsupported listener type: " + listener.getClass().getName());
         }
     }
@@ -72,48 +63,6 @@ public class ListenerSerializers
         }
     };
 
-    private static final IVersionedSerializer<CommandsForRanges.Listener> cfrListener = new IVersionedSerializer<CommandsForRanges.Listener>()
-    {
-        @Override
-        public void serialize(CommandsForRanges.Listener listener, DataOutputPlus out, int version) throws IOException
-        {
-            CommandSerializers.txnId.serialize(listener.txnId, out, version);
-        }
-
-        @Override
-        public CommandsForRanges.Listener deserialize(DataInputPlus in, int version) throws IOException
-        {
-            return new CommandsForRanges.Listener(CommandSerializers.txnId.deserialize(in, version));
-        }
-
-        @Override
-        public long serializedSize(CommandsForRanges.Listener listener, int version)
-        {
-            return CommandSerializers.txnId.serializedSize(listener.txnId, version);
-        }
-    };
-
-    private static final IVersionedSerializer<CommandsForKey.Listener> cfkListener = new IVersionedSerializer<CommandsForKey.Listener>()
-    {
-        @Override
-        public void serialize(CommandsForKey.Listener listener, DataOutputPlus out, int version) throws IOException
-        {
-            PartitionKey.serializer.serialize((PartitionKey) listener.key(), out, version);
-        }
-
-        @Override
-        public CommandsForKey.Listener deserialize(DataInputPlus in, int version) throws IOException
-        {
-            return CommandsForKey.SerializerSupport.listener(PartitionKey.serializer.deserialize(in, version));
-        }
-
-        @Override
-        public long serializedSize(CommandsForKey.Listener listener, int version)
-        {
-            return PartitionKey.serializer.serializedSize((PartitionKey) listener.key(), version);
-        }
-    };
-
     public static final IVersionedSerializer<Command.DurableAndIdempotentListener> listener = new IVersionedSerializer<Command.DurableAndIdempotentListener>()
     {
         @Override
@@ -126,12 +75,6 @@ public class ListenerSerializers
                 case COMMAND:
                     commandListener.serialize((Command.ProxyListener) listener, out, version);
                     break;
-                case COMMANDS_FOR_KEY:
-                    cfkListener.serialize((CommandsForKey.Listener) listener, out, version);
-                    break;
-                case COMMANDS_FOR_RANGE:
-                    cfrListener.serialize((CommandsForRanges.Listener) listener, out, version);
-                    break;
                 default:
                     throw new IllegalArgumentException();
             }
@@ -145,10 +88,6 @@ public class ListenerSerializers
             {
                 case COMMAND:
                     return commandListener.deserialize(in, version);
-                case COMMANDS_FOR_KEY:
-                    return cfkListener.deserialize(in, version);
-                case COMMANDS_FOR_RANGE:
-                    return cfrListener.deserialize(in, version);
                 default:
                     throw new IllegalArgumentException();
             }
@@ -164,12 +103,6 @@ public class ListenerSerializers
                 case COMMAND:
                     size += commandListener.serializedSize((Command.ProxyListener) listener, version);
                     break;
-                case COMMANDS_FOR_KEY:
-                    size += cfkListener.serializedSize((CommandsForKey.Listener) listener, version);
-                    break;
-                case COMMANDS_FOR_RANGE:
-                    size += cfrListener.serializedSize((CommandsForRanges.Listener) listener, version);
-                    break;
                 default:
                     throw new IllegalArgumentException();
             }

src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java

@@ -27,11 +27,14 @@ import accord.messages.ReadData.CommitOrReadNack;
 import accord.messages.ReadData.ReadOk;
 import accord.messages.ReadData.ReadReply;
 import accord.messages.ReadData.ReadType;
+import accord.messages.ReadEphemeralTxnData;
 import accord.messages.ReadTxnData;
 import accord.messages.WaitUntilApplied;
+import accord.primitives.FullRoute;
+import accord.primitives.PartialDeps;
+import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
 import accord.primitives.Ranges;
-import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
@@ -74,15 +77,17 @@ public class ReadDataSerializers
     public static class ApplyThenWaitUntilAppliedSerializer implements ReadDataSerializer<ApplyThenWaitUntilApplied>
     {
         @Override
-        public void serialize(ApplyThenWaitUntilApplied applyThenWaitUntilApplied, DataOutputPlus out, int version) throws IOException
+        public void serialize(ApplyThenWaitUntilApplied msg, DataOutputPlus out, int version) throws IOException
         {
-            CommandSerializers.txnId.serialize(applyThenWaitUntilApplied.txnId, out, version);
-            KeySerializers.partialRoute.serialize(applyThenWaitUntilApplied.route, out, version);
-            DepsSerializer.partialDeps.serialize(applyThenWaitUntilApplied.deps, out, version);
-            KeySerializers.seekables.serialize(applyThenWaitUntilApplied.partialTxnKeys, out, version);
-            CommandSerializers.writes.serialize(applyThenWaitUntilApplied.writes, out, version);
-            TxnResult.serializer.serialize((TxnResult) applyThenWaitUntilApplied.txnResult, out, version);
-            out.writeBoolean(applyThenWaitUntilApplied.notifyAgent);
+            CommandSerializers.txnId.serialize(msg.txnId, out, version);
+            KeySerializers.participants.serialize(msg.readScope, out, version);
+            out.writeUnsignedVInt(msg.executeAtEpoch);
+            KeySerializers.fullRoute.serialize(msg.route, out, version);
+            CommandSerializers.partialTxn.serialize(msg.txn, out, version);
+            DepsSerializer.partialDeps.serialize(msg.deps, out, version);
+            CommandSerializers.writes.serialize(msg.writes, out, version);
+            TxnResult.serializer.serialize((TxnResult) msg.result, out, version);
+            KeySerializers.nullableSeekables.serialize(msg.notify, out, version);
         }
 
         @Override
@@ -90,24 +95,28 @@ public class ReadDataSerializers
         {
             return ApplyThenWaitUntilApplied.SerializerSupport.create(
             CommandSerializers.txnId.deserialize(in, version),
-            KeySerializers.partialRoute.deserialize(in, version),
+            KeySerializers.participants.deserialize(in, version),
+            in.readUnsignedVInt(),
+            KeySerializers.fullRoute.deserialize(in, version),
+            CommandSerializers.partialTxn.deserialize(in, version),
             DepsSerializer.partialDeps.deserialize(in, version),
-            KeySerializers.seekables.deserialize(in, version),
             CommandSerializers.writes.deserialize(in, version),
             TxnResult.serializer.deserialize(in, version),
-            in.readBoolean());
+            KeySerializers.nullableSeekables.deserialize(in, version));
         }
 
         @Override
-        public long serializedSize(ApplyThenWaitUntilApplied applyThenWaitUntilApplied, int version)
+        public long serializedSize(ApplyThenWaitUntilApplied msg, int version)
         {
-            return CommandSerializers.txnId.serializedSize(applyThenWaitUntilApplied.txnId, version)
-                   + KeySerializers.partialRoute.serializedSize(applyThenWaitUntilApplied.route, version)
-                   + DepsSerializer.partialDeps.serializedSize(applyThenWaitUntilApplied.deps, version)
-                   + KeySerializers.seekables.serializedSize(applyThenWaitUntilApplied.partialTxnKeys, version)
-                   + CommandSerializers.writes.serializedSize(applyThenWaitUntilApplied.writes, version)
-                   + TxnResult.serializer.serializedSize((TxnData)applyThenWaitUntilApplied.txnResult, version)
-                   + sizeof(applyThenWaitUntilApplied.notifyAgent);
+            return CommandSerializers.txnId.serializedSize(msg.txnId, version)
+                   + KeySerializers.participants.serializedSize(msg.readScope, version)
+                   + TypeSizes.sizeofUnsignedVInt(msg.executeAtEpoch)
+                   + KeySerializers.fullRoute.serializedSize(msg.route, version)
+                   + CommandSerializers.partialTxn.serializedSize(msg.txn, version)
+                   + DepsSerializer.partialDeps.serializedSize(msg.deps, version)
+                   + CommandSerializers.writes.serializedSize(msg.writes, version)
+                   + TxnResult.serializer.serializedSize((TxnData)msg.result, version)
+                   + KeySerializers.nullableSeekables.serializedSize(msg.notify, version);
         }
     }
 
@@ -118,8 +127,7 @@ public class ReadDataSerializers
         {
             CommandSerializers.txnId.serialize(read.txnId, out, version);
             KeySerializers.participants.serialize(read.readScope, out, version);
-            out.writeUnsignedVInt(read.waitForEpoch());
-            out.writeUnsignedVInt(read.executeAtEpoch - read.waitForEpoch());
+            out.writeUnsignedVInt(read.executeAtEpoch);
         }
 
         @Override
@@ -127,9 +135,8 @@ public class ReadDataSerializers
         {
             TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
             Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
-            long waitForEpoch = in.readUnsignedVInt();
-            long executeAtEpoch = in.readUnsignedVInt() + waitForEpoch;
-            return ReadTxnData.SerializerSupport.create(txnId, readScope, executeAtEpoch, waitForEpoch);
+            long executeAtEpoch = in.readUnsignedVInt();
+            return ReadTxnData.SerializerSupport.create(txnId, readScope, executeAtEpoch);
         }
 
         @Override
@@ -137,8 +144,44 @@ public class ReadDataSerializers
         {
             return CommandSerializers.txnId.serializedSize(read.txnId, version)
                    + KeySerializers.participants.serializedSize(read.readScope, version)
-                   + TypeSizes.sizeofUnsignedVInt(read.waitForEpoch())
-                   + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch - read.waitForEpoch());
+                   + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch);
+        }
+    };
+
+    private static final ReadDataSerializer<ReadEphemeralTxnData> readEphemeralTxnData = new ReadDataSerializer<ReadEphemeralTxnData>()
+    {
+        @Override
+        public void serialize(ReadEphemeralTxnData read, DataOutputPlus out, int version) throws IOException
+        {
+            CommandSerializers.txnId.serialize(read.txnId, out, version);
+            KeySerializers.participants.serialize(read.readScope, out, version);
+            out.writeUnsignedVInt(read.executeAtEpoch);
+            CommandSerializers.partialTxn.serialize(read.partialTxn, out, version);
+            DepsSerializer.partialDeps.serialize(read.partialDeps, out, version);
+            KeySerializers.fullRoute.serialize(read.route, out, version);
+        }
+
+        @Override
+        public ReadEphemeralTxnData deserialize(DataInputPlus in, int version) throws IOException
+        {
+            TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
+            Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
+            long executeAtEpoch = in.readUnsignedVInt();
+            PartialTxn partialTxn = CommandSerializers.partialTxn.deserialize(in, version);
+            PartialDeps partialDeps = DepsSerializer.partialDeps.deserialize(in, version);
+            FullRoute<?> route = KeySerializers.fullRoute.deserialize(in, version);
+            return ReadEphemeralTxnData.SerializerSupport.create(txnId, readScope, executeAtEpoch, partialTxn, partialDeps, route);
+        }
+
+        @Override
+        public long serializedSize(ReadEphemeralTxnData read, int version)
+        {
+            return CommandSerializers.txnId.serializedSize(read.txnId, version)
+                   + KeySerializers.participants.serializedSize(read.readScope, version)
+                   + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
+                   + CommandSerializers.partialTxn.serializedSize(read.partialTxn, version)
+                   + DepsSerializer.partialDeps.serializedSize(read.partialDeps, version)
+                   + KeySerializers.fullRoute.serializedSize(read.route, version);
         }
     };
 
@@ -160,6 +203,8 @@ public class ReadDataSerializers
         {
             case readTxnData:
                 return readTxnData;
+            case readDataWithoutTimestamp:
+                return readEphemeralTxnData;
             case applyThenWaitUntilApplied:
                 return applyThenWaitUntilApplied;
             case waitUntilApplied:
@@ -171,7 +216,7 @@ public class ReadDataSerializers
 
     public static final class ReplySerializer<D extends Data> implements IVersionedSerializer<ReadReply>
     {
-        // TODO (now): use something other than ordinal
+        // TODO (expected): use something other than ordinal
         final CommitOrReadNack[] nacks = CommitOrReadNack.values();
         private final IVersionedSerializer<D> dataSerializer;
 
@@ -185,26 +230,33 @@ public class ReadDataSerializers
         {
             if (!reply.isOk())
             {
-                out.writeByte(1 + ((CommitOrReadNack) reply).ordinal());
+                out.writeByte(2 + ((CommitOrReadNack) reply).ordinal());
                 return;
             }
 
-            out.writeByte(0);
+            boolean isFutureEpochOk = reply.getClass() == ReadData.ReadOkWithFutureEpoch.class;
+            out.writeByte(isFutureEpochOk ? 1 : 0);
             ReadOk readOk = (ReadOk) reply;
             serializeNullable(readOk.unavailable, out, version, KeySerializers.ranges);
             dataSerializer.serialize((D) readOk.data, out, version);
+            if (isFutureEpochOk)
+                out.writeUnsignedVInt(((ReadData.ReadOkWithFutureEpoch) reply).futureEpoch);
         }
 
         @Override
         public ReadReply deserialize(DataInputPlus in, int version) throws IOException
         {
             int id = in.readByte();
-            if (id != 0)
-                return nacks[id - 1];
+            if (id > 1)
+                return nacks[id - 2];
 
-            Ranges ranges = deserializeNullable(in, version, KeySerializers.ranges);
+            Ranges unavailable = deserializeNullable(in, version, KeySerializers.ranges);
             D data = dataSerializer.deserialize(in, version);
-            return new ReadOk(ranges, data);
+            if (id == 0)
+                return new ReadOk(unavailable, data);
+
+            long futureEpoch = in.readUnsignedVInt();
+            return new ReadData.ReadOkWithFutureEpoch(unavailable, data, futureEpoch);
         }
 
         @Override
@@ -214,9 +266,12 @@ public class ReadDataSerializers
                 return TypeSizes.BYTE_SIZE;
 
             ReadOk readOk = (ReadOk) reply;
-            return TypeSizes.BYTE_SIZE
-                   + serializedNullableSize(readOk.unavailable, version, KeySerializers.ranges)
-                   + dataSerializer.serializedSize((D) readOk.data, version);
+            long size = TypeSizes.BYTE_SIZE
+                        + serializedNullableSize(readOk.unavailable, version, KeySerializers.ranges)
+                        + dataSerializer.serializedSize((D) readOk.data, version);
+            if (readOk instanceof ReadData.ReadOkWithFutureEpoch)
+                size += TypeSizes.sizeofUnsignedVInt(((ReadData.ReadOkWithFutureEpoch) readOk).futureEpoch);
+            return size;
         }
     }
 
@@ -230,8 +285,7 @@ public class ReadDataSerializers
         {
             CommandSerializers.txnId.serialize(waitUntilApplied.txnId, out, version);
             KeySerializers.participants.serialize(waitUntilApplied.readScope, out, version);
-            out.writeUnsignedVInt(waitUntilApplied.waitForEpoch());
-            CommandSerializers.timestamp.serialize(waitUntilApplied.executeAt, out , version);
+            out.writeUnsignedVInt(waitUntilApplied.executeAtEpoch);
         }
 
         @Override
@@ -239,9 +293,8 @@ public class ReadDataSerializers
         {
             TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
             Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
-            long waitForEpoch = in.readUnsignedVInt();
-            Timestamp executeAt = CommandSerializers.timestamp.deserialize(in, version);
-            return WaitUntilApplied.SerializerSupport.create(txnId, readScope, executeAt, waitForEpoch);
+            long executeAtEpoch = in.readUnsignedVInt();
+            return WaitUntilApplied.SerializerSupport.create(txnId, readScope, executeAtEpoch);
         }
 
         @Override
@@ -249,8 +302,7 @@ public class ReadDataSerializers
         {
             return CommandSerializers.txnId.serializedSize(waitUntilApplied.txnId, version)
                    + KeySerializers.participants.serializedSize(waitUntilApplied.readScope, version)
-                   + TypeSizes.sizeofUnsignedVInt(waitUntilApplied.waitForEpoch())
-                   + CommandSerializers.timestamp.serializedSize(waitUntilApplied.executeAt, version);
+                   + TypeSizes.sizeofUnsignedVInt(waitUntilApplied.executeAtEpoch);
         }
     };
 }

src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java

@@ -26,7 +26,6 @@ import accord.primitives.Deps;
 import accord.primitives.Seekables;
 import accord.primitives.SyncPoint;
 import accord.primitives.TxnId;
-import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -59,19 +58,21 @@ public class SetDurableSerializers
         @Override
         public void serialize(SetGloballyDurable msg, DataOutputPlus out, int version) throws IOException
         {
+            CommandSerializers.txnId.serialize(msg.txnId, out, version);
             CommandStoreSerializers.durableBefore.serialize(msg.durableBefore, out, version);
         }
 
         @Override
         public SetGloballyDurable deserialize(DataInputPlus in, int version) throws IOException
         {
-            return new SetGloballyDurable(CommandStoreSerializers.durableBefore.deserialize(in, version));
+            return new SetGloballyDurable(CommandSerializers.txnId.deserialize(in, version), CommandStoreSerializers.durableBefore.deserialize(in, version));
         }
 
         @Override
         public long serializedSize(SetGloballyDurable msg, int version)
         {
-            return CommandStoreSerializers.durableBefore.serializedSize(msg.durableBefore, version);
+            return CommandSerializers.txnId.serializedSize(msg.txnId, version)
+                   + CommandStoreSerializers.durableBefore.serializedSize(msg.durableBefore, version);
         }
     };
 
@@ -84,7 +85,6 @@ public class SetDurableSerializers
             DepsSerializer.deps.serialize(sp.waitFor, out, version);
             KeySerializers.seekables.serialize(sp.keysOrRanges, out, version);
             KeySerializers.routingKey.serialize(sp.homeKey, out, version);
-            out.writeBoolean(sp.finishedAsync);
         }
 
         @Override
@@ -94,8 +94,7 @@ public class SetDurableSerializers
             Deps waitFor = DepsSerializer.deps.deserialize(in, version);
             Seekables<?, ?> keysOrRanges = KeySerializers.seekables.deserialize(in, version);
             RoutingKey homeKey = KeySerializers.routingKey.deserialize(in, version);
-            boolean finishedAsync = in.readBoolean();
-            return SyncPoint.SerializationSupport.construct(syncId, waitFor, keysOrRanges, homeKey, finishedAsync);
+            return SyncPoint.SerializationSupport.construct(syncId, waitFor, keysOrRanges, homeKey);
         }
 
         @Override
@@ -104,8 +103,7 @@ public class SetDurableSerializers
             return CommandSerializers.txnId.serializedSize(sp.syncId, version)
                    + DepsSerializer.deps.serializedSize(sp.waitFor, version)
                    + KeySerializers.seekables.serializedSize(sp.keysOrRanges, version)
-                   + KeySerializers.routingKey.serializedSize(sp.homeKey, version)
-                   + TypeSizes.sizeof(sp.finishedAsync);
+                   + KeySerializers.routingKey.serializedSize(sp.homeKey, version);
         }
     };
 }

src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java

@@ -375,7 +375,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         // TODO (expected, efficiency): 99.9999% of the time we can just use executeAt.hlc(), so can avoid bringing
         //  cfk into memory by retaining at all times in memory key ranges that are dirty and must use this logic;
         //  any that aren't can just use executeAt.hlc
-        TimestampsForKey cfk = CommandsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?,?,?,?>) safeStore, (RoutableKey) key, executeAt, true);
+        TimestampsForKey cfk = CommandsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?,?,?>) safeStore, (RoutableKey) key, executeAt, true);
         long timestamp = AccordSafeTimestampsForKey.timestampMicrosFor(cfk, executeAt, true);
         // TODO (low priority - do we need to compute nowInSeconds, or can we just use executeAt?)
         int nowInSeconds = AccordSafeTimestampsForKey.nowInSecondsFor(cfk, executeAt, true);

src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java

@@ -211,7 +211,7 @@ public class BlockingReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.Fo
             Future<TxnResult> repairFuture;
             try
             {
-                Txn txn = new Txn.InMemory(key, TxnRead.createNoOpRead(key), TxnQuery.NONE, repairUpdate);
+                Txn txn = new Txn.InMemory(Txn.Kind.Read, key, TxnRead.createNoOpRead(key), TxnQuery.NONE, repairUpdate);
                 repairFuture = Stage.ACCORD_MIGRATION.submit(() -> {
                     try
                     {

src/java/org/apache/cassandra/utils/ByteBufferUtil.java

@@ -45,6 +45,7 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import net.nicoulaj.compilecommand.annotations.DontInline;
 import net.nicoulaj.compilecommand.annotations.Inline;
 
 import org.apache.cassandra.db.TypeSizes;
@@ -1014,6 +1015,104 @@ public class ByteBufferUtil
         }
     }
 
+    public static void writeLeastSignificantBytes(long register, int bytes, ByteBuffer out)
+    {
+        writeMostSignificantBytesSlow(register << ((8 - bytes)*8), bytes, out);
+    }
+
+    public static void writeMostSignificantBytes(long register, int bytes, ByteBuffer out)
+    {
+        int position = out.position();
+        int limit = out.limit();
+        if (limit - position < Long.BYTES)
+        {
+            writeMostSignificantBytesSlow(register, bytes, out);
+        }
+        else
+        {
+            out.putLong(position, register);
+            out.position(position + bytes);
+        }
+    }
+
+    @DontInline
+    private static void writeMostSignificantBytesSlow(long register, int bytes, ByteBuffer out)
+    {
+        switch (bytes)
+        {
+            case 0:
+                break;
+            case 1:
+                out.put((byte)(register >>> 56));
+                break;
+            case 2:
+                out.putShort((short)(register >> 48));
+                break;
+            case 3:
+                out.putShort((short)(register >> 48));
+                out.put((byte)(register >> 40));
+                break;
+            case 4:
+                out.putInt((int)(register >> 32));
+                break;
+            case 5:
+                out.putInt((int)(register >> 32));
+                out.put((byte)(register >> 24));
+                break;
+            case 6:
+                out.putInt((int)(register >> 32));
+                out.putShort((short)(register >> 16));
+                break;
+            case 7:
+                out.putInt((int)(register >> 32));
+                out.putShort((short)(register >> 16));
+                out.put((byte)(register >> 8));
+                break;
+            case 8:
+                out.putLong(register);
+                break;
+            default:
+                throw new IllegalArgumentException();
+        }
+    }
+
+    public static long readLeastSignificantBytes(int bytes, ByteBuffer in)
+    {
+        if (bytes == 0)
+            return 0L;
+
+        int position = in.position();
+        int limit = in.limit();
+        if (limit - position < Long.BYTES)
+        {
+            return readLeastSignificantBytesSlow(bytes, in);
+        }
+        else
+        {
+            long result = in.getLong(position);
+            in.position(position + bytes);
+            return result >>> (64 - 8*bytes);
+        }
+    }
+
+    @DontInline
+    private static long readLeastSignificantBytesSlow(int bytes, ByteBuffer out)
+    {
+        switch (bytes)
+        {
+            case 0: return 0;
+            case 1: return out.get() & 0xffL;
+            case 2: return out.getShort() & 0xffffL;
+            case 3: return ((out.getShort() & 0xffffL) << 8) | (out.get() & 0xffL);
+            case 4: return out.getInt() & 0xffffffffL;
+            case 5: return ((out.getInt() & 0xffffffffL) << 8) | (out.get() & 0xffL);
+            case 6: return ((out.getInt() & 0xffffffffL) << 16) | (out.getShort() & 0xffffL);
+            case 7: return ((out.getInt() & 0xffffffffL) << 24) | ((out.getShort() & 0xffffL) << 8) | (out.get() & 0xffL);
+            case 8: return out.getLong();
+            default: throw new IllegalArgumentException();
+        }
+    }
+
     public static final IVersionedSerializer<ByteBuffer> byteBufferSerializer = new IVersionedSerializer<ByteBuffer>()
     {
         @Override

src/java/org/apache/cassandra/utils/vint/VIntCoding.java

@@ -49,8 +49,10 @@ package org.apache.cassandra.utils.vint;
 import java.io.DataInput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 
 import io.netty.util.concurrent.FastThreadLocal;
+import net.nicoulaj.compilecommand.annotations.DontInline;
 import net.nicoulaj.compilecommand.annotations.Inline;
 import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -109,6 +111,50 @@ public class VIntCoding
         return retval;
     }
 
+    @DontInline
+    private static long readUnsignedVIntSlow(ByteBuffer in, byte firstByte)
+    {
+        int size = numberOfExtraBytesToRead(firstByte);
+        long retval = firstByte & firstByteValueMask(size);
+        for (int ii = 0; ii < size; ii++)
+        {
+            byte b = in.get();
+            retval <<= 8;
+            retval |= b & 0xff;
+        }
+
+        return retval;
+    }
+
+    public static long readUnsignedVInt(ByteBuffer in)
+    {
+        byte firstByte = in.get();
+        if (firstByte >= 0)
+            return firstByte;
+
+
+        int position = in.position();
+        int limit = in.limit();
+        if (limit - position < 8)
+            return readUnsignedVIntSlow(in, firstByte);
+
+        int extraBytes = VIntCoding.numberOfExtraBytesToRead(firstByte);
+        int extraBits = extraBytes * 8;
+
+        long retval = in.getLong(position);
+        if (in.order() == ByteOrder.LITTLE_ENDIAN)
+            retval = Long.reverseBytes(retval);
+        in.position(position + extraBytes);
+
+        // truncate the bytes we read in excess of those we needed
+        retval >>>= 64 - extraBits;
+        // remove the non-value bits from the first byte
+        firstByte &= VIntCoding.firstByteValueMask(extraBytes);
+        // shift the first byte up to its correct position
+        retval |= (long) firstByte << extraBits;
+        return retval;
+    }
+
     public static void skipUnsignedVInt(DataInputPlus input) throws IOException
     {
         int firstByte = input.readByte();
@@ -271,6 +317,19 @@ public class VIntCoding
         return checkedCast(readUnsignedVInt(input));
     }
 
+    /**
+     * Read up to a 32-bit integer.
+     *
+     * This method assumes the original integer was written using {@link #writeUnsignedVInt32(int, DataOutputPlus)}
+     * or similar that doesn't zigzag encodes the vint.
+     *
+     * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer
+     */
+    public static int readUnsignedVInt32(ByteBuffer input)
+    {
+        return checkedCast(readUnsignedVInt(input));
+    }
+
     // & this with the first byte to give the value part for a given extraBytesToRead encoded in the byte
     public static int firstByteValueMask(int extraBytesToRead)
     {

test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java

@@ -155,7 +155,7 @@ public class AccordMigrationTest extends AccordTestBase
         StorageService.instance.setPartitionerUnsafe(partitioner);
         ServerTestUtils.prepareServerNoRegister();
         minToken = partitioner.getMinimumToken();
-        maxToken = partitioner.getMaximumToken();
+        maxToken = partitioner.getMaximumTokenForSplitting();
         midToken = partitioner.midpoint(minToken, maxToken);
         upperMidToken = partitioner.midpoint(midToken, maxToken);
         lowerMidToken = partitioner.midpoint(minToken, midToken);

test/simulator/main/org/apache/cassandra/simulator/paxos/StrictSerializabilityValidator.java

@@ -32,7 +32,7 @@ public class StrictSerializabilityValidator implements HistoryValidator
 
     public StrictSerializabilityValidator(int[] primaryKeys)
     {
-        this.verifier = new StrictSerializabilityVerifier(primaryKeys.length);
+        this.verifier = new StrictSerializabilityVerifier("", primaryKeys.length);
         pkToIndex = new IntIntHashMap(primaryKeys.length);
         indexToPk = new int[primaryKeys.length];
         for (int i = 0; i < primaryKeys.length; i++)

test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java

@@ -186,7 +186,7 @@ public class DiskBoundaryManagerTest extends CQLTester
 
         SSTableReader disk1Boundary = MockSchema.sstable(gen++, (long)sstableFirstDisk1.getTokenValue(), (long)tokens.get(0).getTokenValue(), 0, mock);
         SSTableReader disk2Full = MockSchema.sstable(gen++, (long)tokens.get(0).nextValidToken().getTokenValue(), (long)tokens.get(1).getTokenValue(), 0, mock);
-        SSTableReader disk3Full = MockSchema.sstable(gen++, (long)tokens.get(1).nextValidToken().getTokenValue(), (long)partitioner.getMaximumToken().getTokenValue(), 0, mock);
+        SSTableReader disk3Full = MockSchema.sstable(gen++, (long)tokens.get(1).nextValidToken().getTokenValue(), (long)partitioner.getMaximumTokenForSplitting().getTokenValue(), 0, mock);
 
         Assert.assertEquals(tableDirs, mock.getDirectoriesForFiles(ImmutableSet.of()));
         Assert.assertEquals(Lists.newArrayList(tableDirs.get(0)), mock.getDirectoriesForFiles(ImmutableSet.of(containedDisk1)));

test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java

@@ -464,7 +464,7 @@ public class CancelCompactionsTest extends CQLTester
         try (LifecycleTransaction txn = idx.getTracker().tryModify(idx.getLiveSSTables(), OperationType.COMPACTION))
         {
             IPartitioner partitioner = getCurrentColumnFamilyStore().getPartitioner();
-            getCurrentColumnFamilyStore().forceCompactionForTokenRange(Collections.singleton(new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken())));
+            getCurrentColumnFamilyStore().forceCompactionForTokenRange(Collections.singleton(new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting())));
         }
     }
 

test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java

@@ -29,7 +29,6 @@ import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -37,8 +36,11 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import accord.api.Key;
 import accord.api.Result;
+import accord.impl.CommandsForKey;
 import accord.local.CheckedCommands;
+import accord.local.Command;
 import accord.local.CommandStore;
 import accord.local.DurableBefore;
 import accord.local.RedundantBefore;
@@ -85,10 +87,13 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordTestUtils;
 import org.apache.cassandra.service.accord.IAccordService;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
 import static accord.impl.TimestampsForKey.NO_LAST_EXECUTED_HLC;
+import static accord.local.KeyHistory.COMMANDS;
 import static accord.local.PreLoadContext.contextFor;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
 import static org.apache.cassandra.Util.spinAssertEquals;
@@ -123,8 +128,7 @@ public class CompactionAccordIteratorsTest
 
     static ColumnFamilyStore commands;
     static ColumnFamilyStore timestampsForKey;
-    static ColumnFamilyStore depsCommandsForKey;
-    static ColumnFamilyStore allCommandsForKey;
+    static ColumnFamilyStore commandsForKey;
     static TableMetadata table;
     static FullRoute<?> route;
     Random random;
@@ -144,17 +148,14 @@ public class CompactionAccordIteratorsTest
                                     parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks"));
         StorageService.instance.initServer();
 
-        commands = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, COMMANDS);
+        commands = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.COMMANDS);
         commands.disableAutoCompaction();
 
         timestampsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, TIMESTAMPS_FOR_KEY);
         timestampsForKey.disableAutoCompaction();
 
-        depsCommandsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, DEPS_COMMANDS_FOR_KEY);
-        depsCommandsForKey.disableAutoCompaction();
-
-        allCommandsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, ALL_COMMANDS_FOR_KEY);
-        allCommandsForKey.disableAutoCompaction();
+        commandsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, COMMANDS_FOR_KEY);
+        commandsForKey.disableAutoCompaction();
 
         table = ColumnFamilyStore.getIfExists("ks", "tbl").metadata();
         route = AccordTestUtils.keys(table, 42).toRoute(AccordTestUtils.key(table, 42).toUnseekable());
@@ -209,7 +210,7 @@ public class CompactionAccordIteratorsTest
     {
         testWithCommandStore((commandStore) -> {
             IAccordService mockAccordService = mockAccordService(commandStore, redundantBefore, durableBefore);
-            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, COMMANDS);
+            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, AccordKeyspace.COMMANDS);
             List<Partition> result = compactCFS(mockAccordService, cfs);
             expectedResult.accept(result);
         }, false);
@@ -249,7 +250,6 @@ public class CompactionAccordIteratorsTest
             Partition partition = partitions.get(0);
             Row row = partition.getRow(Clustering.EMPTY);
 
-            assertEquals(SECOND_TXN_ID, TimestampsForKeyRows.getMaxTimestamp(row));
             assertEquals(TXN_ID, TimestampsForKeyRows.getLastExecutedTimestamp(row));
             assertEquals(TXN_ID, TimestampsForKeyRows.getLastWriteTimestamp(row));
 
@@ -264,27 +264,17 @@ public class CompactionAccordIteratorsTest
         return partitions -> {
             assertEquals(1, partitions.size());
             Partition partition = partitions.get(0);
-            assertEquals(2, Iterators.size(partition.unfilteredIterator()));
-            UnfilteredRowIterator rows = partition.unfilteredIterator();
-            // One row per txn per series
-            for (TxnId txnId : TXN_IDS)
-                assertEquals(txnId, DepsCommandsForKeysAccessor.getTimestamp((Row)rows.next()));
+            PartitionKey partitionKey = new PartitionKey(partition.metadata().id, partition.partitionKey());
+            CommandsForKey cfk = CommandsForKeysAccessor.getCommandsForKey(partitionKey, ((Row)partition.unfilteredIterator().next()));
+            assertEquals(TXN_IDS.length, cfk.size());
+            for (int i = 0 ; i < TXN_IDS.length ; ++i)
+                assertEquals(TXN_IDS[i], cfk.txnId(i));
         };
     }
 
     private static Consumer<List<Partition>> expectedAccordTimestampsForKeyEraseOne()
     {
-        return partitions -> {
-            assertEquals(1, partitions.size());
-            Partition partition = partitions.get(0);
-            Row row = partition.getRow(Clustering.EMPTY);
-            // Only expect one column to remain because the second transaction is a read
-            assertEquals(1, Iterables.size(row));
-            assertEquals(SECOND_TXN_ID, AccordKeyspace.TimestampsForKeyRows.getMaxTimestamp(row));
-            assertNull(AccordKeyspace.TimestampsForKeyRows.getLastExecutedTimestamp(row));
-            assertNull(AccordKeyspace.TimestampsForKeyRows.getLastWriteTimestamp(row));
-            assertEquals(NO_LAST_EXECUTED_HLC, AccordKeyspace.TimestampsForKeyRows.getLastExecutedMicros(row));
-        };
+        return partitions -> assertEquals(0, partitions.size());
     }
 
     private static Consumer<List<Partition>> expectedAccordCommandsForKeyEraseOne()
@@ -294,7 +284,7 @@ public class CompactionAccordIteratorsTest
             Partition partition = partitions.get(0);
             assertEquals(1, Iterators.size(partition.unfilteredIterator()));
             UnfilteredRowIterator rows = partition.unfilteredIterator();
-            assertEquals(TXN_IDS[1], DepsCommandsForKeysAccessor.getTimestamp((Row)rows.next()));
+//            assertEquals(TXN_IDS[1], CommandsForKeysAccessor.getTimestamp((Row)rows.next()));
         };
     }
 
@@ -322,7 +312,7 @@ public class CompactionAccordIteratorsTest
     {
         testWithCommandStore((commandStore) -> {
             IAccordService mockAccordService = mockAccordService(commandStore, redundantBefore, DurableBefore.EMPTY);
-            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, DEPS_COMMANDS_FOR_KEY);
+            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, COMMANDS_FOR_KEY);
             List<Partition> result = compactCFS(mockAccordService, cfs);
             expectedResult.accept(result);
         }, true);
@@ -447,8 +437,7 @@ public class CompactionAccordIteratorsTest
         });
         commands.forceBlockingFlush(FlushReason.UNIT_TESTS);
         timestampsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
-        depsCommandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
-        allCommandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
+        commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
     }
 
     private void testWithCommandStore(TestWithCommandStore test, boolean additionalCommand) throws Throwable
@@ -457,62 +446,68 @@ public class CompactionAccordIteratorsTest
         clock.set(CLOCK_START);
         AccordCommandStore commandStore = AccordTestUtils.createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
         TxnId[] txnIds = additionalCommand ? TXN_IDS : new TxnId[] {TXN_ID};
+        Txn writeTxn = AccordTestUtils.createWriteTxn(42);
+        Txn readTxn = AccordTestUtils.createTxn(42);
+        Seekable key = writeTxn.keys().get(0);
         for (TxnId txnId : txnIds)
         {
-            Txn txn = txnId.kind().isWrite() ? AccordTestUtils.createWriteTxn(42) : AccordTestUtils.createTxn(42);
-            Seekable key = txn.keys().get(0);
+            Txn txn = txnId.kind().isWrite() ? writeTxn : readTxn;
             PartialDeps partialDeps = Deps.NONE.slice(AccordTestUtils.fullRange(txn));
             PartialTxn partialTxn = txn.slice(commandStore.unsafeRangesForEpoch().currentRanges(), true);
             PartialRoute<?> partialRoute = route.slice(commandStore.unsafeRangesForEpoch().currentRanges());
-            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys()), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys(), COMMANDS), safe -> {
                 PreAccept preAccept =
                     PreAccept.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), txnId.epoch(), false, txnId.epoch(), partialTxn, route);
                 commandStore.appendToJournal(preAccept);
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, null);
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys()), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys(), COMMANDS), safe -> {
                 Accept accept =
                     Accept.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), txnId.epoch(), false, Ballot.ZERO, txnId, partialTxn.keys(), partialDeps);
                 commandStore.appendToJournal(accept);
                 CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, partialTxn.keys(), null, txnId, partialDeps);
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys()), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys(), COMMANDS), safe -> {
                 Commit commit =
-                    Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableFastPath, Ballot.ZERO, txnId, partialTxn, partialDeps, route, null);
+                    Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableFastPath, Ballot.ZERO, txnId, partialTxn.keys(), partialTxn, partialDeps, route, null);
                 commandStore.appendToJournal(commit);
                 CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, null, partialTxn, txnId, partialDeps);
             }).beginAsResult());
             flush(commandStore);
-            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys()), safe -> {
+            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys(), COMMANDS), safe -> {
                 Pair<Writes, Result> result = AccordTestUtils.processTxnResultDirect(safe, txnId, partialTxn, txnId);
                 Apply apply =
-                    Apply.SerializationSupport.create(txnId, partialRoute, txnId.epoch(), Apply.Kind.Minimal, partialTxn.keys(), txnId, partialDeps, partialTxn, result.left, result.right);
+                    Apply.SerializationSupport.create(txnId, partialRoute, txnId.epoch(), Apply.Kind.Minimal, partialTxn.keys(), txnId, partialDeps, partialTxn, null, result.left, result.right);
                 commandStore.appendToJournal(apply);
                 CheckedCommands.apply(safe, txnId, route, null, txnId, partialDeps, partialTxn, result.left, result.right);
             }).beginAsResult());
+            getUninterruptibly(commandStore.execute(contextFor(txnId, txn.keys(), COMMANDS), safe -> {
+                safe.get(txnId, txnId, route).addListener(new Command.ProxyListener(txnId)); // add a junk listener just to test it in compaction
+            }).beginAsResult());
             flush(commandStore);
             // The apply chain is asychronous, so it is easiest to just spin until it is applied
             // in order to have the updated state in the system table
             spinAssertEquals(true, 5, () ->
-                                      getUninterruptibly(commandStore.submit(contextFor(txnId, txn.keys()), safe -> safe.get(txnId, route.homeKey()).current().hasBeen(Status.Applied)
+                                      getUninterruptibly(commandStore.submit(contextFor(txnId, txn.keys(), COMMANDS), safe -> safe.get(txnId, route.homeKey()).current().hasBeen(Status.Applied)
                                       ).beginAsResult()));
             flush(commandStore);
         }
 
-        UntypedResultSet commandsTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + COMMANDS + ";");
+        UntypedResultSet commandsTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + AccordKeyspace.COMMANDS + ";");
         logger.info(commandsTable.toStringUnsafe());
         assertEquals(txnIds.length, commandsTable.size());
         Iterator<UntypedResultSet.Row> commandsTableIterator = commandsTable.iterator();
         for (TxnId txnId : txnIds)
             assertEquals(txnId, AccordKeyspace.deserializeTimestampOrNull(commandsTableIterator.next().getBytes("txn_id"), TxnId::fromBits));
-        UntypedResultSet commandsForKeyTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + DEPS_COMMANDS_FOR_KEY + ";");
+        UntypedResultSet commandsForKeyTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + COMMANDS_FOR_KEY + ";");
         logger.info(commandsForKeyTable.toStringUnsafe());
-        assertEquals(txnIds.length, commandsForKeyTable.size());
-        Iterator<UntypedResultSet.Row> commandsForKeyTableIterator = commandsTable.iterator();
-        for (TxnId txnId : txnIds)
-            assertEquals(txnId, AccordKeyspace.deserializeTimestampOrNull(commandsForKeyTableIterator.next().getBytes("txn_id"), TxnId::fromBits));
+        assertEquals(1, commandsForKeyTable.size());
+        CommandsForKey cfk = CommandsForKeySerializer.fromBytes((Key)key, commandsForKeyTable.iterator().next().getBytes("data"));
+        assertEquals(txnIds.length, cfk.size());
+        for (int i = 0 ; i < txnIds.length ; ++i)
+            assertEquals(txnIds[i], cfk.txnId(i));
         test.test(commandStore);
     }
 
@@ -525,7 +520,7 @@ public class CompactionAccordIteratorsTest
         {
             List<Partition> outputPartitions = new ArrayList<>();
             List<ISSTableScanner> nextInputScanners = new ArrayList<>();
-            if (singleCompaction)
+            if (singleCompaction || numScanners == 1)
             {
                 nextInputScanners = ImmutableList.copyOf(scanners);
                 scanners.clear();
@@ -554,7 +549,7 @@ public class CompactionAccordIteratorsTest
                 scanners.add(random.nextInt(scanners.size()), new Scanner(cfs.metadata(), outputPartitions.stream().map(Partition::unfilteredIterator).collect(Collectors.toList())));
         } while (!scanners.isEmpty());
 
-        verify(mockAccordService, times(singleCompaction ? 1 : numScanners - 1)).getRedundantBeforesAndDurableBefore();
+        verify(mockAccordService, times(singleCompaction || numScanners == 1 ? 1 : numScanners - 1)).getRedundantBeforesAndDurableBefore();
         return result;
     }
 }

test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java

@@ -179,7 +179,7 @@ public class CompactionsBytemanTest extends CQLTester
     {
         testStopCompactionRepaired((cfs) -> {
             Collection<Range<Token>> ranges = Collections.singleton(new Range<>(cfs.getPartitioner().getMinimumToken(),
-                                                                                cfs.getPartitioner().getMaximumToken()));
+                                                                                cfs.getPartitioner().getMaximumTokenForSplitting()));
             CompactionManager.instance.forceCompactionForTokenRange(cfs, ranges);
         });
     }

test/unit/org/apache/cassandra/db/compaction/ShardManagerTest.java

@@ -543,7 +543,7 @@ public class ShardManagerTest
 
     private Token boundary(int numSSTables, int i)
     {
-        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), i * 1.0 / numSSTables);
+        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting(), i * 1.0 / numSSTables);
     }
 
     private SSTableReader mockSSTable(DecoratedKey first, DecoratedKey last)

test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java

@@ -277,7 +277,7 @@ public class UnifiedCompactionStrategyTest
 
         IPartitioner partitioner = cfs.getPartitioner();
         DecoratedKey first = new BufferDecoratedKey(partitioner.getMinimumToken(), ByteBuffer.allocate(0));
-        DecoratedKey last = new BufferDecoratedKey(partitioner.getMaximumToken(), ByteBuffer.allocate(0));
+        DecoratedKey last = new BufferDecoratedKey(partitioner.getMaximumTokenForSplitting(), ByteBuffer.allocate(0));
 
         List<SSTableReader> sstables = new ArrayList<>();
         long dataSetSizeBytes = 0;
@@ -517,7 +517,7 @@ public class UnifiedCompactionStrategyTest
     {
         List<SSTableReader> mockSSTables = new ArrayList<>();
         Token min = partitioner.getMinimumToken();
-        Token max = partitioner.getMaximumToken();
+        Token max = partitioner.getMaximumTokenForSplitting();
         ByteBuffer bb = ByteBuffer.allocate(0);
         sstablesMap.forEach((size, num) -> {
             Token first = min.getPartitioner().split(min, max, 0.01);
@@ -1118,7 +1118,7 @@ public class UnifiedCompactionStrategyTest
 
     private Token boundary(int numSSTables, double i)
     {
-        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumToken(), i / numSSTables);
+        return partitioner.split(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting(), i / numSSTables);
     }
 
 }

test/unit/org/apache/cassandra/dht/LengthPartitioner.java

@@ -76,7 +76,7 @@ public class LengthPartitioner extends AccordSplitter implements IPartitioner
     }
 
     @Override
-    public Token getMaximumToken()
+    public Token getMaximumTokenForSplitting()
     {
         return null;
     }

test/unit/org/apache/cassandra/dht/SplitterTest.java

@@ -168,7 +168,7 @@ public class SplitterTest
 
         for (int i = 0; i < tokens.size(); i++)
         {
-            Token end = i == tokens.size() - 1 ? partitioner.getMaximumToken() : tokens.get(i);
+            Token end = i == tokens.size() - 1 ? partitioner.getMaximumTokenForSplitting() : tokens.get(i);
             splits.add(sumOwnedBetween(localRanges, start, end, splitter, splitIndividualRanges));
             start = end;
         }
@@ -250,7 +250,7 @@ public class SplitterTest
         boolean isRandom = partitioner instanceof RandomPartitioner;
         Splitter splitter = getSplitter(partitioner);
         BigInteger min = splitter.valueForToken(partitioner.getMinimumToken());
-        BigInteger max = splitter.valueForToken(partitioner.getMaximumToken());
+        BigInteger max = splitter.valueForToken(partitioner.getMaximumTokenForSplitting());
         BigInteger first = isRandom ? RandomPartitioner.ZERO : min;
         BigInteger last = isRandom ? max.subtract(BigInteger.valueOf(1)) : max;
         BigInteger midpoint = last.add(first).divide(BigInteger.valueOf(2));
@@ -373,8 +373,8 @@ public class SplitterTest
         Splitter splitter = getSplitter(partitioner);
 
         // test full range
-        Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
-        BigInteger fullRangeSize = splitter.valueForToken(partitioner.getMaximumToken()).subtract(splitter.valueForToken(partitioner.getMinimumToken()));
+        Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting());
+        BigInteger fullRangeSize = splitter.valueForToken(partitioner.getMaximumTokenForSplitting()).subtract(splitter.valueForToken(partitioner.getMinimumToken()));
         assertEquals(fullRangeSize, splitter.tokensInRange(fullRange));
         fullRange = new Range<>(splitter.tokenForValue(BigInteger.valueOf(-10)), splitter.tokenForValue(BigInteger.valueOf(-10)));
         assertEquals(fullRangeSize, splitter.tokensInRange(fullRange));
@@ -413,13 +413,13 @@ public class SplitterTest
 
         // wrapped range
         BigInteger min = splitter.valueForToken(partitioner.getMinimumToken());
-        BigInteger max = splitter.valueForToken(partitioner.getMaximumToken());
+        BigInteger max = splitter.valueForToken(partitioner.getMaximumTokenForSplitting());
         Range<Token> wrappedRange = new Range<>(splitter.tokenForValue(max.subtract(BigInteger.valueOf(1350))),
                                                 splitter.tokenForValue(min.add(BigInteger.valueOf(20394))));
         testElapsedTokens(partitioner, wrappedRange, true);
 
         // full range
-        Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+        Range<Token> fullRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting());
         testElapsedTokens(partitioner, fullRange, false);
     }
 
@@ -490,15 +490,15 @@ public class SplitterTest
         testPositionInRange(partitioner, splitter, range);
 
         // Test wrap-around range
-        start = splitter.tokenForValue(splitter.valueForToken(partitioner.getMaximumToken()).subtract(BigInteger.valueOf(123456789)));
+        start = splitter.tokenForValue(splitter.valueForToken(partitioner.getMaximumTokenForSplitting()).subtract(BigInteger.valueOf(123456789)));
         end = splitter.tokenForValue(splitter.valueForToken(partitioner.getMinimumToken()).add(BigInteger.valueOf(123456789)));
         range = new Range<>(start, end);
         testPositionInRange(partitioner, splitter, range);
 
         // Test full range
-        testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+        testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting()));
         testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken()));
-        testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMaximumToken(), partitioner.getMaximumToken()));
+        testPositionInRange(partitioner, splitter, new Range<>(partitioner.getMaximumTokenForSplitting(), partitioner.getMaximumTokenForSplitting()));
         testPositionInRange(partitioner, splitter, new Range<>(splitter.tokenForValue(BigInteger.ONE), splitter.tokenForValue(BigInteger.ONE)));
     }
 
@@ -508,7 +508,7 @@ public class SplitterTest
         //full range case
         if (range.left.equals(range.right))
         {
-            actualRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+            actualRange = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting());
         }
         assertEquals(0.0, splitter.positionInRange(actualRange.left, range), 0.01);
         assertEquals(0.25, splitter.positionInRange(getTokenInPosition(partitioner, actualRange, 0.25), range), 0.01);
@@ -523,7 +523,7 @@ public class SplitterTest
     {
         if (range.left.equals(range.right))
         {
-            range = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken());
+            range = new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting());
         }
         Splitter splitter = getSplitter(partitioner);
         BigInteger totalTokens = splitter.tokensInRange(range);
@@ -535,7 +535,7 @@ public class SplitterTest
     private static Token getWrappedToken(IPartitioner partitioner, BigInteger position)
     {
         Splitter splitter = getSplitter(partitioner);
-        BigInteger maxTokenValue = splitter.valueForToken(partitioner.getMaximumToken());
+        BigInteger maxTokenValue = splitter.valueForToken(partitioner.getMaximumTokenForSplitting());
         BigInteger minTokenValue = splitter.valueForToken(partitioner.getMinimumToken());
         if (position.compareTo(maxTokenValue) > 0)
         {

test/unit/org/apache/cassandra/index/sai/cql/VectorTypeTest.java

@@ -551,7 +551,7 @@ public class VectorTypeTest extends VectorTester
     private Collection<Integer> keysWithLowerBound(Collection<Integer> keys, int leftKey, boolean leftInclusive)
     {
         return keysInTokenRange(keys, partitioner.getToken(Int32Type.instance.decompose(leftKey)), leftInclusive,
-                                partitioner.getMaximumToken().getToken(), true);
+                                partitioner.getMaximumTokenForSplitting().getToken(), true);
     }
 
     private Collection<Integer> keysWithUpperBound(Collection<Integer> keys, int rightKey, boolean rightInclusive)

test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java

@@ -185,7 +185,7 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
                                                                     0,
                                                                     Long.MAX_VALUE,
                                                                     SAITester.TEST_FACTORY.create(DatabaseDescriptor.getPartitioner().getMinimumToken()),
-                                                                    SAITester.TEST_FACTORY.create(DatabaseDescriptor.getPartitioner().getMaximumToken()),
+                                                                    SAITester.TEST_FACTORY.create(DatabaseDescriptor.getPartitioner().getMaximumTokenForSplitting()),
                                                                     wrap(termsEnum.get(0).left),
                                                                     wrap(termsEnum.get(terms - 1).left),
                                                                     indexMetas);

test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentTest.java

@@ -50,7 +50,7 @@ public class SegmentTest
         DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
         partitioner = DatabaseDescriptor.getPartitioner();
         min = partitioner.getMinimumToken();
-        max = partitioner.getMaximumToken();
+        max = partitioner.getMaximumTokenForSplitting();
         tokens = IntStream.rangeClosed(0, 10).boxed().map(i -> partitioner.getRandomToken())
                           .distinct().sorted().collect(Collectors.toList());
     }

test/unit/org/apache/cassandra/repair/RepairJobTest.java

@@ -106,7 +106,7 @@ public class RepairJobTest
     private static final Range<Token> RANGE_3 = range(4, 5);
     private static final RepairJobDesc JOB_DESC = new RepairJobDesc(nextTimeUUID(), nextTimeUUID(), KEYSPACE, CF, Collections.emptyList());
     private static final List<Range<Token>> FULL_RANGE = Collections.singletonList(new Range<>(MURMUR3_PARTITIONER.getMinimumToken(),
-                                                                                               MURMUR3_PARTITIONER.getMaximumToken()));
+                                                                                               MURMUR3_PARTITIONER.getMaximumTokenForSplitting()));
     private static InetAddressAndPort addr1;
     private static InetAddressAndPort addr2;
     private static InetAddressAndPort addr3;

test/unit/org/apache/cassandra/repair/ValidationTaskTest.java

@@ -67,7 +67,7 @@ public class ValidationTaskTest
 
         IPartitioner partitioner = Murmur3Partitioner.instance;
         MerkleTrees trees = new MerkleTrees(partitioner);
-        trees.addMerkleTree(128, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumToken()));
+        trees.addMerkleTree(128, new Range<>(partitioner.getMinimumToken(), partitioner.getMaximumTokenForSplitting()));
         task.treesReceived(trees);
         assertEquals(1, trees.size());
         

test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java

@@ -18,12 +18,10 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.nio.ByteBuffer;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicLong;
 
-import com.google.common.collect.ImmutableSortedMap;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -33,23 +31,18 @@ import org.slf4j.LoggerFactory;
 
 import accord.api.Key;
 import accord.api.Result;
-import accord.impl.CommandTimeseries;
 import accord.impl.CommandsForKey;
 import accord.impl.CommandsForKeys;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.CommonAttributes;
-import accord.local.KeyHistory;
-import accord.local.PreLoadContext;
 import accord.local.SaveStatus;
 import accord.messages.Apply;
 import accord.primitives.Ballot;
-import accord.primitives.Keys;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialTxn;
 import accord.primitives.Ranges;
 import accord.primitives.Route;
-import accord.primitives.RoutableKey;
 import accord.primitives.RoutingKeys;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
@@ -65,10 +58,8 @@ import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordCachingState.Modified;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
-import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 import org.apache.cassandra.utils.Pair;
 
 import static accord.local.Status.Durability.Majority;
@@ -157,6 +148,7 @@ public class AccordCommandStoreTest
                                               executeAt,
                                               dependencies,
                                               txn,
+                                              null,
                                               result.left,
                                               CommandSerializers.APPLIED);
         commandStore.appendToJournal(apply);
@@ -186,7 +178,6 @@ public class AccordCommandStoreTest
 
         AccordSafeTimestampsForKey tfk = new AccordSafeTimestampsForKey(loaded(key, null));
         tfk.initialize();
-        tfk.updateMax(maxTimestamp);
 
         CommandsForKeys.updateLastExecutionTimestamps(commandStore, tfk, txnId1, true);
         Assert.assertEquals(txnId1.hlc(), AccordSafeTimestampsForKey.timestampMicrosFor(tfk.current(), txnId1, true));
@@ -198,13 +189,10 @@ public class AccordCommandStoreTest
         Assert.assertEquals(txnId2.hlc(), tfk.lastExecutedMicros());
 
         AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null));
-        cfk.initialize(CommandsForKeySerializer.loader);
+        cfk.initialize();
 
-        AccordSafeCommandsForKeyUpdate ufk = new AccordSafeCommandsForKeyUpdate(loaded(key, null));
-        ufk.initialize();
-
-        CommandsForKeys.registerCommand(tfk, ufk, command1);
-        CommandsForKeys.registerCommand(tfk, ufk, command2);
+        cfk.set(cfk.current().update(null, command1));
+        cfk.set(cfk.current().update(null, command2));
 
         AccordKeyspace.getTimestampsForKeyMutation(commandStore, tfk, commandStore.nextSystemTimestampMicros()).apply();
         logger.info("E: {}", tfk);
@@ -232,20 +220,17 @@ public class AccordCommandStoreTest
         tfk.initialize();
 
         AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null));
-        cfk.initialize(CommandsForKeySerializer.loader);
+        cfk.initialize();
 
-        AccordSafeCommandsForKeyUpdate ufk = new AccordSafeCommandsForKeyUpdate(loaded(key, null));
-        ufk.initialize();
+        cfk.set(cfk.current().update(null, command1));
+        cfk.set(cfk.current().update(null, command2));
 
-        CommandsForKeys.registerCommand(tfk, ufk, command1);
-        CommandsForKeys.registerCommand(tfk, ufk, command2);
-
-        AccordKeyspace.getCommandsForKeyMutation(commandStore.id(), ufk.setUpdates(), commandStore.nextSystemTimestampMicros()).apply();
+        AccordKeyspace.getCommandsForKeyMutation(commandStore.id(), cfk.current(), commandStore.nextSystemTimestampMicros()).apply();
         logger.info("E: {}", cfk);
-        CommandsForKey actual = AccordKeyspace.loadDepsCommandsForKey(commandStore, key);
+        CommandsForKey actual = AccordKeyspace.loadCommandsForKey(commandStore, key);
         logger.info("A: {}", actual);
 
-        Assert.assertEquals(ufk.applyToDeps(cfk.current()), actual);
+        Assert.assertEquals(cfk.current(), actual);
     }
 
     private static <K, V extends AccordSafeState<K, ?>> NavigableMap<K, V> toNavigableMap(V safeState)
@@ -254,109 +239,4 @@ public class AccordCommandStoreTest
         map.put(safeState.key(), safeState);
         return map;
     }
-
-    @Test
-    public void commandsForKeyUpdateTest()
-    {
-        // check that updates are reflected in CFKs without marking them modified
-        AtomicLong clock = new AtomicLong(0);
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-
-        PartialTxn txn = createPartialTxn(1);
-        PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
-
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-
-        AccordSafeCommand safeCommand = commandStore.commandCache().acquireOrInitialize(txnId, t -> preaccepted(txnId, txn, timestamp(1, clock.incrementAndGet(), 1)));
-        AccordSafeTimestampsForKey timestamps = commandStore.timestampsForKeyCache().acquireOrInitialize(key, k -> new TimestampsForKey((Key) k));
-        AccordSafeCommandsForKey commands = commandStore.depsCommandsForKeyCache().acquireOrInitialize(key, k -> new CommandsForKey((Key) k, CommandsForKeySerializer.loader));
-        AccordSafeCommandsForKeyUpdate update = commandStore.updatesForKeyCache().acquireOrInitialize(key, CommandsForKeyUpdate::empty);
-
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.commandCache().getUnsafe(txnId).status());
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.timestampsForKeyCache().getUnsafe(key).status());
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.depsCommandsForKeyCache().getUnsafe(key).status());
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.updatesForKeyCache().getUnsafe(key).status());
-
-        AccordSafeCommandStore safeStore = commandStore.beginOperation(PreLoadContext.contextFor(txnId, Keys.of(key), KeyHistory.DEPS),
-                                                                       toNavigableMap(safeCommand),
-                                                                       toNavigableMap(timestamps),
-                                                                       toNavigableMap(commands),
-                                                                       new TreeMap<>(),
-                                                                       toNavigableMap(update));
-
-        AccordSafeCommandsForKeyUpdate updates = safeStore.getOrCreateCommandsForKeyUpdate(key);
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.updatesForKeyCache().getUnsafe(key).status());
-
-        Command initialCommand = safeCommand.current();
-        CommandsForKey initialCFK = commands.current();
-        CommandsForKeyUpdate initialUpdate = updates.current();
-
-        updates.common().commands().add(txnId, initialCommand);
-
-        CommandsForKeyUpdate expected = new CommandsForKeyUpdate(key, updates.deps().toImmutable(), updates.all().toImmutable(), updates.common().toImmutable());
-        Assert.assertEquals(1, expected.common().commands().numChanges());
-        Assert.assertTrue(expected.deps().isEmpty());
-        Assert.assertTrue(expected.all().isEmpty());
-
-        Assert.assertSame(initialCFK, commands.current());
-        Assert.assertSame(initialUpdate, updates.current());
-
-        safeStore.postExecute(toNavigableMap(safeCommand),
-                              toNavigableMap(timestamps),
-                              toNavigableMap(commands),
-                              new TreeMap<>(),
-                              toNavigableMap(updates));
-        safeStore.complete();
-
-
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.commandCache().getUnsafe(txnId).status());
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.timestampsForKeyCache().getUnsafe(key).status());
-        Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.depsCommandsForKeyCache().getUnsafe(key).status());
-        Assert.assertEquals(AccordCachingState.Status.MODIFIED, commandStore.updatesForKeyCache().getUnsafe(key).status());
-
-        CommandsForKey finalCFK = commandStore.depsCommandsForKeyCache().getUnsafe(key).get();
-        Assert.assertEquals(txnId, getOnlyElement(finalCFK.commands().commands.keySet()));
-
-        Modified<RoutableKey, CommandsForKeyUpdate> loadedUpdate = (Modified<RoutableKey, CommandsForKeyUpdate>)  commandStore.updatesForKeyCache().getUnsafe(key).state();
-        Assert.assertNull(loadedUpdate.original);
-        Assert.assertEquals(expected, loadedUpdate.get());
-    }
-
-    /**
-     * Test that in memory cfk updates are applied to
-     */
-    @Test
-    public void commandsForKeyUpdateOnLoadTest()
-    {
-        AtomicLong clock = new AtomicLong(0);
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
-
-        PartialTxn txn = createPartialTxn(1);
-        PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
-
-        TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
-
-        Command command = preaccepted(txnId, txn, timestamp(1, clock.incrementAndGet(), 1));
-
-        // make a cached update
-        AccordSafeCommandsForKeyUpdate updates = commandStore.updatesForKeyCache().acquireOrInitialize(key, k -> null);
-        updates.preExecute();
-        updates.common().commands().remove(command.txnId());
-        updates.setUpdates(); // apply the updates applied to the safe state to the cached value
-        updates.postExecute();  // apply the cached value to the global state
-        commandStore.updatesForKeyCache().release(updates);
-
-        // make an out of date CFK
-        CommandTimeseries.CommandLoader<ByteBuffer> loader = CommandsForKeySerializer.loader;
-        CommandsForKey staleCFK = CommandsForKey.SerializerSupport.create(key, loader,
-                                                                          ImmutableSortedMap.of(command.txnId(), loader.saveForCFK(command)));
-
-        Assert.assertEquals(txnId, getOnlyElement(staleCFK.commands().commands.keySet()));
-
-        // on loading the cfk into the cache, the in memory update should be applied
-        AccordSafeCommandsForKey commands = commandStore.depsCommandsForKeyCache().acquireOrInitialize(key, k -> new CommandsForKey((Key) k, CommandsForKeySerializer.loader));
-        commands.preExecute();
-
-        Assert.assertEquals(txnId, getOnlyElement(staleCFK.commands().commands.keySet()));
-    }
 }

test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java

@@ -27,7 +27,6 @@ import org.junit.Test;
 import accord.api.Key;
 import accord.api.RoutingKey;
 import accord.impl.CommandsForKey;
-import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.KeyHistory;
 import accord.local.Node;
@@ -118,11 +117,8 @@ public class AccordCommandTest
             Assert.assertEquals(Status.PreAccepted, command.status());
             Assert.assertTrue(command.partialDeps() == null || command.partialDeps().isEmpty());
 
-            TimestampsForKey tfk = ((AccordSafeCommandStore) instance).timestampsForKey(key(1)).current();
-            Assert.assertEquals(txnId, tfk.max());
-
-            CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
-            Assert.assertNotNull((cfk.commands()).get(txnId));
+            CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
+            Assert.assertTrue(cfk.indexOf(txnId) >= 0);
         }));
 
         // check accept
@@ -149,26 +145,23 @@ public class AccordCommandTest
             Assert.assertEquals(Status.Accepted, command.status());
             Assert.assertEquals(deps, command.partialDeps());
 
-            TimestampsForKey tfk = ((AccordSafeCommandStore) instance).timestampsForKey(key(1)).current();
-            Assert.assertEquals(executeAt, tfk.max());
-
-            CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
-            Assert.assertNotNull((cfk.commands()).get(txnId));
+            CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
+            Assert.assertTrue(cfk.indexOf(txnId) >= 0);
         }));
 
         // check commit
-        Commit commit = Commit.SerializerSupport.create(txnId, route, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn, deps, fullRoute, null);
+        Commit commit = Commit.SerializerSupport.create(txnId, route, 1, Commit.Kind.StableWithTxnAndDeps, Ballot.ZERO, executeAt, partialTxn.keys(), partialTxn, deps, fullRoute, null);
         commandStore.appendToJournal(commit);
         getUninterruptibly(commandStore.execute(commit, commit::apply));
 
-        getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key), KeyHistory.DEPS), instance -> {
+        getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key), KeyHistory.COMMANDS), instance -> {
             Command command = instance.ifInitialised(txnId).current();
             Assert.assertEquals(commit.executeAt, command.executeAt());
             Assert.assertTrue(command.hasBeen(Status.Committed));
             Assert.assertEquals(commit.partialDeps, command.partialDeps());
 
-            CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
-            Assert.assertNotNull((cfk.commands()).get(txnId));
+            CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
+            Assert.assertTrue(cfk.indexOf(txnId) >= 0);
         }));
     }
 

test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java

@@ -88,7 +88,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
         AccordSafeCommand safeCommand = new AccordSafeCommand(AccordTestUtils.loaded(id, null));
         safeCommand.set(committed);
 
-        Commit commit = Commit.SerializerSupport.create(id, route.slice(scope), 1, Commit.Kind.StableFastPath, Ballot.ZERO, id, partialTxn, partialDeps, route, null);
+        Commit commit = Commit.SerializerSupport.create(id, route.slice(scope), 1, Commit.Kind.StableFastPath, Ballot.ZERO, id, partialTxn.keys(), partialTxn, partialDeps, route, null);
         store.appendToJournal(commit);
 
         Mutation mutation = AccordKeyspace.getCommandMutation(store, safeCommand, 42);

test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java

@@ -21,6 +21,8 @@ package org.apache.cassandra.service.accord;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import accord.messages.ReadData;
+import accord.messages.ReadData.CommitOrReadNack;
 import accord.topology.TopologyUtils;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
@@ -32,7 +34,6 @@ import accord.impl.IntKey;
 import accord.local.Node;
 import accord.messages.InformOfTxnId;
 import accord.messages.MessageType;
-import accord.messages.ReadData;
 import accord.messages.ReadTxnData;
 import accord.messages.Reply;
 import accord.messages.Request;
@@ -88,11 +89,11 @@ public class AccordMessageSinkTest
         Txn txn = Utils.readTxn(Keys.of(IntKey.key(42)));
         TxnId id = nextTxnId(epoch, txn);
         PartialTxn partialTxn = txn.slice(Ranges.of(IntKey.range(40, 50)), true);
-        Request request = new AbstractFetchCoordinator.FetchRequest(epoch, id, partialTxn.covering(), PartialDeps.NONE, partialTxn, true);
+        Request request = new AbstractFetchCoordinator.FetchRequest(epoch, id, partialTxn.covering(), PartialDeps.NONE, partialTxn);
 
         checkRequestReplies(request,
                             new AbstractFetchCoordinator.FetchResponse(null, null, id),
-                            ReadData.CommitOrReadNack.Insufficient);
+                            CommitOrReadNack.Insufficient);
 
     }
 
@@ -100,10 +101,10 @@ public class AccordMessageSinkTest
     public void txnRead()
     {
         TxnId txnId = nextTxnId(42, Txn.Kind.Read, Routable.Domain.Key);
-        Request request = new ReadTxnData(node, topologies, txnId, topology.ranges(), txnId);
+        Request request = new ReadTxnData(node, topologies, txnId, topology.ranges(), txnId.epoch());
         checkRequestReplies(request,
                             new ReadData.ReadOk(null, null),
-                            ReadData.CommitOrReadNack.Insufficient);
+                            CommitOrReadNack.Insufficient);
     }
 
     private static void checkRequestReplies(Request request, Reply... replies)

test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java

@@ -35,11 +35,9 @@ import com.google.common.collect.Sets;
 import org.junit.Assert;
 
 import accord.api.Data;
-import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.api.Result;
 import accord.api.RoutingKey;
-import accord.impl.CommandsForKey;
 import accord.impl.InMemoryCommandStore;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -87,7 +85,6 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.utils.Pair;
@@ -152,11 +149,6 @@ public class AccordTestUtils
         }
     }
 
-    public static CommandsForKey commandsForKey(Key key)
-    {
-        return new CommandsForKey(key, CommandsForKeySerializer.loader);
-    }
-
     public static <K, V> AccordCachingState<K, V> loaded(K key, V value, int index)
     {
         AccordCachingState<K, V> global = new AccordCachingState<>(key, index);

test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java

@@ -165,9 +165,9 @@ public class AccordTopologyTest
     {
         List<Range<Token>> ranges = ImmutableList.of(range(partitioner.getMinimumToken(), token(-100)),
                                                      range(-100, 100),
-                                                     range(token(100), partitioner.getMaximumToken()));
+                                                     range(token(100), partitioner.getMaximumTokenForSplitting()));
         Assert.assertEquals(partitioner.getMinimumToken(), ranges.get(0).left);
-        Assert.assertEquals(partitioner.getMaximumToken(), ranges.get(2).right);
+        Assert.assertEquals(partitioner.getMaximumTokenForSplitting(), ranges.get(2).right);
         ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
 
         Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true);
@@ -203,7 +203,7 @@ public class AccordTopologyTest
     {
         List<Range<Token>> ranges = ImmutableList.of(range(partitioner.getMinimumToken(), token(-100)),
                                                      range(-100, 100),
-                                                     range(token(100), partitioner.getMaximumToken()));
+                                                     range(token(100), partitioner.getMaximumTokenForSplitting()));
         ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
         Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true);
         Topology expected = new Topology(1,
@@ -234,7 +234,7 @@ public class AccordTopologyTest
     {
         List<Range<Token>> ranges = ImmutableList.of(range(partitioner.getMinimumToken(), token(-100)),
                                                      range(-100, 100),
-                                                     range(token(100), partitioner.getMaximumToken()));
+                                                     range(token(100), partitioner.getMaximumTokenForSplitting()));
         ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
         Topology topology = AccordTopology.createAccordTopology(metadata, ks -> true);
         Topology expected = new Topology(1,

test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java

@@ -78,7 +78,7 @@ public class AccordKeyTest
         PartitionKey pk = new PartitionKey(TABLE1, dk);
         TokenKey tk = new TokenKey(TABLE1, dk.getToken());
         TokenKey tkLow = new TokenKey(TABLE1, dk.getToken().decreaseSlightly());
-        TokenKey tkHigh = new TokenKey(TABLE1, dk.getToken().nextValidToken());
+        TokenKey tkHigh = new TokenKey(TABLE1, dk.getToken().increaseSlightly());
 
         Assert.assertTrue(tk.compareTo(pk) > 0);
         Assert.assertTrue(tkLow.compareTo(pk) < 0);

test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java

@@ -18,9 +18,12 @@
 
 package org.apache.cassandra.service.accord.async;
 
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
@@ -29,6 +32,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import accord.api.Key;
+import accord.impl.CommandsForKey;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.KeyHistory;
@@ -47,14 +51,15 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordKeyspace;
 import org.apache.cassandra.service.accord.AccordCachingState;
 import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForKeyUpdate;
+import org.apache.cassandra.service.accord.AccordSafeState;
 import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
 import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.CommandsForKeyUpdate;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.async.AsyncOperation.Context;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 
+import static accord.local.KeyHistory.COMMANDS;
+import static accord.local.KeyHistory.TIMESTAMPS;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
@@ -108,7 +113,7 @@ public class AsyncLoaderTest
         testLoad(executor, safeTimestamps, new TimestampsForKey(key));
         timestampsCache.release(safeTimestamps);
 
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.NONE);
+        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), TIMESTAMPS);
 
         // everything is cached, so the loader should return immediately
         commandStore.executeBlocking(() -> {
@@ -148,7 +153,7 @@ public class AsyncLoaderTest
         AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), null, timestamps.current(), commandStore.nextSystemTimestampMicros()).apply();
 
         // resources are on disk only, so the loader should suspend...
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.DEPS);
+        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), TIMESTAMPS);
         AsyncPromise<Void> cbFired = new AsyncPromise<>();
         Context context = new Context();
         commandStore.executeBlocking(() -> {
@@ -196,7 +201,7 @@ public class AsyncLoaderTest
         AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), null, new TimestampsForKey(key), commandStore.nextSystemTimestampMicros()).apply();
 
         // resources are on disk only, so the loader should suspend...
-        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.NONE);
+        AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), TIMESTAMPS);
         AsyncPromise<Void> cbFired = new AsyncPromise<>();
         Context context = new Context();
         commandStore.executeBlocking(() -> {
@@ -234,17 +239,10 @@ public class AsyncLoaderTest
             createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
         commandStore.executor().submit(() -> commandStore.setCapacity(1024)).get();
         AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
-        AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
         TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
         PartialTxn txn = createPartialTxn(0);
         PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
 
-        // acquire / release
-        timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((PartitionKey) k));
-        AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquire(key);
-        testLoad(executor, safeTimestamps, new TimestampsForKey(key));
-        timestampsCache.release(safeTimestamps);
-
         commandCache.unsafeSetLoadFunction(id -> { Assert.assertEquals(txnId, id); return notDefined(id, txn); });
         AccordSafeCommand safeCommand = commandCache.acquire(txnId);
         Assert.assertEquals(AccordCachingState.Status.LOADING, safeCommand.globalStatus());
@@ -260,7 +258,7 @@ public class AsyncLoaderTest
             boolean result = loader.load(context, (o, t) -> {
                 Assert.assertNull(t);
                 Assert.assertTrue(context.commands.containsKey(txnId));
-                Assert.assertTrue(context.timestampsForKey.containsKey(key));
+                Assert.assertFalse(context.timestampsForKey.containsKey(key));
                 cbFired.setSuccess(null);
             });
             Assert.assertFalse(result);
@@ -276,7 +274,7 @@ public class AsyncLoaderTest
         commandStore.executeBlocking(() -> {
             boolean result = loader.load(context, (o, t) -> Assert.fail());
             Assert.assertTrue(context.commands.containsKey(txnId));
-            Assert.assertTrue(context.timestampsForKey.containsKey(key));
+            Assert.assertFalse(context.timestampsForKey.containsKey(key));
             Assert.assertTrue(result);
         });
     }
@@ -306,7 +304,7 @@ public class AsyncLoaderTest
                 throw new AssertionError("Unknown txnId: " + txnId);
             });
 
-            AsyncLoader loader = new AsyncLoader(commandStore, ImmutableList.of(txnId1, txnId2), Keys.EMPTY, KeyHistory.DEPS);
+            AsyncLoader loader = new AsyncLoader(commandStore, ImmutableList.of(txnId1, txnId2), Keys.EMPTY, KeyHistory.COMMANDS);
 
             boolean result = loader.load(new Context(), (u, t) -> {
                 Assert.assertFalse(callback.isDone());
@@ -376,19 +374,25 @@ public class AsyncLoaderTest
 
     @Test
     public void inProgressCFKSaveTest()
+    {
+        inProgressCFKSaveTest(COMMANDS, AccordCommandStore::commandsForKeyCache, context -> context.commandsForKey, CommandsForKey::new, (cfk, u) -> cfk.update(null, u));
+    }
+
+    @Test
+    public void inProgressTFKSaveTest()
+    {
+        inProgressCFKSaveTest(TIMESTAMPS, AccordCommandStore::timestampsForKeyCache, context -> context.timestampsForKey, TimestampsForKey::new, (tfk, c) -> new TimestampsForKey(tfk.key(), c.executeAt(), c.executeAt().hlc(), c.executeAt()));
+    }
+
+    private <T1, T2 extends AccordSafeState<RoutableKey, T1>, C extends AccordStateCache.Instance<RoutableKey, T1, T2>>  void inProgressCFKSaveTest(KeyHistory history, Function<AccordCommandStore, C> getter, Function<Context, TreeMap<?, ?>> inContext, Function<Key, T1> initialiser, BiFunction<T1, Command, T1> update)
     {
         AtomicLong clock = new AtomicLong(0);
         ManualExecutor executor = new ManualExecutor();
         AccordCommandStore commandStore =
         createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
 
-        AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
-        timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((PartitionKey) k));
-        timestampsCache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
-
-        AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updateCache = commandStore.updatesForKeyCache();
-        updateCache.unsafeSetLoadFunction(k -> { throw new AssertionError("updates shouldn't be loaded"); });
-        updateCache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
+        C cache = getter.apply(commandStore);
+        cache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
 
         TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
         PartialTxn txn = createPartialTxn(0);
@@ -396,42 +400,37 @@ public class AsyncLoaderTest
         Command preaccepted = preaccepted(txnId, txn, txnId);
 
         // acquire / release
+        T2 safe = cache.acquireOrInitialize(key, k -> initialiser.apply((Key)k));
+        safe.preExecute();
+        safe.set(update.apply(safe.current(), preaccepted));
+        cache.release(safe);
 
-        AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquireOrInitialize(key, k -> new TimestampsForKey((Key) k));
-        timestampsCache.release(safeTimestamps);
-        Assert.assertEquals(AccordCachingState.Status.LOADED, timestampsCache.getUnsafe(key).status());
-
-        AccordSafeCommandsForKeyUpdate safeUpdate = updateCache.acquireOrInitialize(key, CommandsForKeyUpdate::empty);
-        safeUpdate.common().commands().add(txnId, preaccepted);
-        safeUpdate.setUpdates();
-        updateCache.release(safeUpdate);
-
-        Assert.assertEquals(AccordCachingState.Status.MODIFIED, updateCache.getUnsafe(key).status());
-        updateCache.getUnsafe(key).save(executor, (before, after) -> () -> {});
-        Assert.assertEquals(AccordCachingState.Status.SAVING, updateCache.getUnsafe(key).status());
+        Assert.assertEquals(AccordCachingState.Status.MODIFIED, cache.getUnsafe(key).status());
+        cache.getUnsafe(key).save(executor, (before, after) -> () -> {});
+        Assert.assertEquals(AccordCachingState.Status.SAVING, cache.getUnsafe(key).status());
 
         // since the command is still saving, the loader shouldn't be able to acquire a reference
-        AsyncLoader loader = new AsyncLoader(commandStore, emptyList(), Keys.of(key), KeyHistory.NONE);
+        AsyncLoader loader = new AsyncLoader(commandStore, emptyList(), Keys.of(key), history);
         AsyncPromise<Void> cbFired = new AsyncPromise<>();
         Context context = new Context();
         commandStore.executeBlocking(() -> {
             boolean result = loader.load(context, (o, t) -> {
                 Assert.assertNull(t);
-                Assert.assertTrue(context.timestampsForKey.containsKey(key));
+                Assert.assertEquals(context.timestampsForKey.containsKey(key), inContext.apply(context) == context.timestampsForKey);
+                Assert.assertEquals(context.commandsForKey.containsKey(key), inContext.apply(context) == context.commandsForKey);
                 cbFired.setSuccess(null);
             });
             Assert.assertFalse(result);
         });
 
-        Assert.assertEquals(AccordCachingState.Status.SAVING, updateCache.getUnsafe(key).status());
         executor.runOne();
         cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
-        Assert.assertEquals(AccordCachingState.Status.LOADED, updateCache.getUnsafe(key).status());
 
         // then return immediately after the callback has fired
         commandStore.executeBlocking(() -> {
             boolean result = loader.load(context, (o, t) -> Assert.fail());
-            Assert.assertTrue(context.timestampsForKey.containsKey(key));
+            Assert.assertEquals(context.timestampsForKey.containsKey(key), inContext.apply(context) == context.timestampsForKey);
+            Assert.assertEquals(context.commandsForKey.containsKey(key), inContext.apply(context) == context.commandsForKey);
             Assert.assertTrue(result);
         });
     }

test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java

@@ -85,6 +85,7 @@ import org.assertj.core.api.Assertions;
 import org.awaitility.Awaitility;
 import org.mockito.Mockito;
 
+import static accord.local.KeyHistory.COMMANDS;
 import static accord.local.PreLoadContext.contextFor;
 import static accord.utils.Property.qt;
 import static accord.utils.async.AsyncChains.getUninterruptibly;
@@ -115,8 +116,7 @@ public class AsyncOperationTest
     {
         QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.COMMANDS));
         QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.TIMESTAMPS_FOR_KEY));
-        QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.DEPS_COMMANDS_FOR_KEY));
-        QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.ALL_COMMANDS_FOR_KEY));
+        QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.COMMANDS_FOR_KEY));
     }
 
     /**
@@ -150,12 +150,12 @@ public class AsyncOperationTest
         PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
 
         getUninterruptibly(commandStore.execute(contextFor(key), instance -> {
-            SafeCommandsForKey cfk = ((AccordSafeCommandStore) instance).maybeDepsCommandsForKey(key);
+            SafeCommandsForKey cfk = ((AccordSafeCommandStore) instance).maybeCommandsForKey(key);
             Assert.assertNull(cfk);
         }));
 
         long nowInSeconds = FBUtilities.nowInSeconds();
-        SinglePartitionReadCommand command = AccordKeyspace.getDepsCommandsForKeyRead(commandStore.id(), key, (int) nowInSeconds);
+        SinglePartitionReadCommand command = AccordKeyspace.getCommandsForKeyRead(commandStore.id(), key, (int) nowInSeconds);
         try(ReadExecutionController controller = command.executionController();
             FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
         {
@@ -177,6 +177,7 @@ public class AsyncOperationTest
                                             Commit.Kind.StableWithTxnAndDeps,
                                             Ballot.ZERO,
                                             executeAt,
+                                            command.partialTxn().keys(),
                                             command.partialTxn(),
                                             command.partialDeps(),
                                             Route.castToFullRoute(command.route()),
@@ -209,14 +210,14 @@ public class AsyncOperationTest
         PreAccept preAccept =
             PreAccept.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), txnId.epoch(), false, txnId.epoch(), partialTxn, route);
         Commit stable =
-            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableFastPath, Ballot.ZERO, executeAt, partialTxn, deps, route, null);
+            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableFastPath, Ballot.ZERO, executeAt, partialTxn.keys(), partialTxn, deps, route, null);
 
         commandStore.appendToJournal(preAccept);
         commandStore.appendToJournal(stable);
 
         try
         {
-            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, partialTxn.keys()), safe -> {
+            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, partialTxn.keys(), COMMANDS), safe -> {
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, null);
                 CheckedCommands.commit(safe, SaveStatus.Stable, Ballot.ZERO, txnId, route, null, partialTxn, executeAt, deps);
                 return safe.ifInitialised(txnId).current();
@@ -258,9 +259,9 @@ public class AsyncOperationTest
         Accept accept =
             Accept.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), txnId.epoch(), false, Ballot.ZERO, executeAt, partialTxn.keys(), deps);
         Commit commit =
-            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.Commit, Ballot.ZERO, executeAt, partialTxn, deps, route, null);
+            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.Commit, Ballot.ZERO, executeAt, partialTxn.keys(), partialTxn, deps, route, null);
         Commit stable =
-            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableSlowPath, Ballot.ZERO, executeAt, partialTxn, deps, route, null);
+            Commit.SerializerSupport.create(txnId, partialRoute, txnId.epoch(), Commit.Kind.StableSlowPath, Ballot.ZERO, executeAt, partialTxn.keys(), partialTxn, deps, route, null);
 
         commandStore.appendToJournal(preAccept);
         commandStore.appendToJournal(accept);
@@ -269,7 +270,7 @@ public class AsyncOperationTest
 
         try
         {
-            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, partialTxn.keys()), safe -> {
+            Command command = getUninterruptibly(commandStore.submit(contextFor(txnId, partialTxn.keys(), COMMANDS), safe -> {
                 CheckedCommands.preaccept(safe, txnId, partialTxn, route, null);
                 CheckedCommands.accept(safe, txnId, Ballot.ZERO, partialRoute, partialTxn.keys(), null, executeAt, deps);
                 CheckedCommands.commit(safe, SaveStatus.Committed, Ballot.ZERO, txnId, route, null, partialTxn, executeAt, deps);
@@ -320,7 +321,7 @@ public class AsyncOperationTest
 
         createStableAndPersist(commandStore, txnId);
 
-        Consumer<SafeCommandStore> consumer = safeStore -> safeStore.ifInitialised(txnId).readyToExecute();
+        Consumer<SafeCommandStore> consumer = safeStore -> safeStore.ifInitialised(txnId).readyToExecute(safeStore);
         PreLoadContext ctx = contextFor(txnId);
         AsyncOperation<Void> operation = new AsyncOperation.ForConsumer(commandStore, ctx, consumer)
         {
@@ -380,7 +381,7 @@ public class AsyncOperationTest
 
             assertNoReferences(commandStore, ids, keys);
 
-            PreLoadContext ctx = contextFor(ids, keys);
+            PreLoadContext ctx = contextFor(null, ids, keys, COMMANDS);
 
             Consumer<SafeCommandStore> consumer = Mockito.mock(Consumer.class);
 
@@ -406,7 +407,7 @@ public class AsyncOperationTest
 
             // can we recover?
             commandStore.commandCache().unsafeSetLoadFunction(txnId -> AccordKeyspace.loadCommand(commandStore, txnId));
-            AsyncOperation.ForConsumer o2 = new AsyncOperation.ForConsumer(commandStore, ctx, store -> ids.forEach(id -> store.ifInitialised(id).readyToExecute()));
+            AsyncOperation.ForConsumer o2 = new AsyncOperation.ForConsumer(commandStore, ctx, store -> ids.forEach(id -> store.ifInitialised(id).readyToExecute(store)));
             getUninterruptibly(o2);
         });
     }
@@ -428,7 +429,7 @@ public class AsyncOperationTest
             createCommand(commandStore, rs, ids);
             assertNoReferences(commandStore, ids, keys);
 
-            PreLoadContext ctx = contextFor(ids, keys);
+            PreLoadContext ctx = contextFor(null, ids, keys, COMMANDS);
 
             Consumer<SafeCommandStore> consumer = Mockito.mock(Consumer.class);
             String errorMsg = "txn_ids " + ids;
@@ -482,7 +483,7 @@ public class AsyncOperationTest
         try
         {
             //TODO this is due to bad typing for Instance, it doesn't use ? extends RoutableKey
-            assertNoReferences(commandStore.depsCommandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
+            assertNoReferences(commandStore.commandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
         }
         catch (AssertionError e)
         {
@@ -524,7 +525,7 @@ public class AsyncOperationTest
     {
         awaitDone(commandStore.commandCache(), ids);
         //TODO this is due to bad typing for Instance, it doesn't use ? extends RoutableKey
-        awaitDone(commandStore.depsCommandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
+        awaitDone(commandStore.commandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
     }
 
     private static <T> void awaitDone(AccordStateCache.Instance<T, ?, ?> cache, Iterable<T> keys)

test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java

@@ -19,26 +19,70 @@
 package org.apache.cassandra.service.accord.serializers;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.BooleanSupplier;
+import java.util.function.Function;
+import java.util.function.IntSupplier;
+import java.util.function.LongUnaryOperator;
+import java.util.function.Supplier;
 
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import accord.impl.CommandTimeseries;
+import accord.api.Key;
+import accord.impl.CommandsForKey;
+import accord.impl.CommandsForKey.InternalStatus;
+import accord.local.Command;
+import accord.local.CommonAttributes;
+import accord.local.CommonAttributes.Mutable;
+import accord.local.Listeners;
+import accord.local.Node;
+import accord.local.SaveStatus;
+import accord.local.Status;
+import accord.primitives.Ballot;
+import accord.primitives.KeyDeps;
+import accord.primitives.PartialDeps;
+import accord.primitives.PartialTxn;
+import accord.primitives.RangeDeps;
+import accord.primitives.Routable;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
 import accord.primitives.TxnId;
+import accord.primitives.Writes;
 import accord.utils.AccordGens;
 import accord.utils.Gens;
+import accord.utils.RandomSource;
+import accord.utils.SortedArrays;
+import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordTestUtils;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.service.accord.txn.TxnData;
+import org.apache.cassandra.service.accord.txn.TxnWrite;
+import org.apache.cassandra.simulator.RandomSource.Choices;
 import org.apache.cassandra.utils.AccordGenerators;
+import org.apache.cassandra.utils.CassandraGenerators;
 
+import static accord.local.Status.Durability.NotDurable;
+import static accord.local.Status.KnownExecuteAt.ExecuteAtErased;
+import static accord.local.Status.KnownExecuteAt.ExecuteAtUnknown;
 import static accord.utils.Property.qt;
+import static accord.utils.SortedArrays.Search.FAST;
 import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn;
 
 public class CommandsForKeySerializerTest
 {
@@ -52,45 +96,396 @@ public class CommandsForKeySerializerTest
         StorageService.instance.initServer();
     }
 
-    @Test
-    public void serdeDeps()
+    static class Cmd
     {
-        DataOutputBuffer buffer = new DataOutputBuffer();
-        int version = MessagingService.Version.VERSION_40.value;
-        qt().forAll(Gens.lists(AccordGens.txnIds()).ofSizeBetween(0, 10)).check(ids -> {
-            buffer.clear();
+        final TxnId txnId;
+        final SaveStatus saveStatus;
+        final PartialTxn txn;
+        final Timestamp executeAt;
+        final List<TxnId> deps = new ArrayList<>();
+        final List<TxnId> missing = new ArrayList<>();
+        boolean invisible;
 
-            long expectedSize = CommandsForKeySerializer.depsIdSerializer.serializedSize(ids, version);
+        Cmd(TxnId txnId, PartialTxn txn, SaveStatus saveStatus, Timestamp executeAt)
+        {
+            this.txnId = txnId;
+            this.saveStatus = saveStatus;
+            this.txn = txn;
+            this.executeAt = executeAt;
+        }
 
-            CommandsForKeySerializer.depsIdSerializer.serialize(ids, buffer, version);
-            assertThat(buffer.position()).isEqualTo(expectedSize);
-            try (DataInputBuffer in = new DataInputBuffer(buffer.unsafeGetBufferAndFlip(), false))
+        CommonAttributes attributes()
+        {
+            Mutable mutable = new Mutable(txnId);
+            if (saveStatus.known.isDefinitionKnown())
+                mutable.partialTxn(txn);
+
+            mutable.route(txn.keys().toRoute(txn.keys().get(0).someIntersectingRoutingKey(null)));
+            mutable.durability(NotDurable);
+            if (saveStatus.known.deps.hasProposedOrDecidedDeps())
             {
-                List<TxnId> read = CommandsForKeySerializer.depsIdSerializer.deserialize(in, version);
-                assertThat(read).isEqualTo(ids);
+                try (KeyDeps.Builder builder = KeyDeps.builder();)
+                {
+                    for (TxnId id : deps)
+                        builder.add((Key)txn.keys().get(0), id);
+                    mutable.partialDeps(new PartialDeps(AccordTestUtils.fullRange(txn), builder.build(), RangeDeps.NONE));
+                }
             }
-        });
+
+            return mutable;
+        }
+
+        Command toCommand()
+        {
+            switch (saveStatus)
+            {
+                default: throw new AssertionError("Unhandled saveStatus: " + saveStatus);
+                case Uninitialised:
+                case NotDefined:
+                    return Command.SerializerSupport.notDefined(attributes(), Ballot.ZERO);
+                case PreAccepted:
+                    return Command.SerializerSupport.preaccepted(attributes(), executeAt, Ballot.ZERO);
+                case Accepted:
+                case AcceptedInvalidate:
+                case AcceptedWithDefinition:
+                case AcceptedInvalidateWithDefinition:
+                case PreCommittedWithDefinition:
+                case PreCommittedWithDefinitionAndAcceptedDeps:
+                case PreCommittedWithAcceptedDeps:
+                case PreCommitted:
+                    return Command.SerializerSupport.accepted(attributes(), saveStatus, executeAt, Ballot.ZERO, Ballot.ZERO);
+
+                case Committed:
+                    return Command.SerializerSupport.committed(attributes(), saveStatus, executeAt, Ballot.ZERO, Ballot.ZERO, null);
+
+                case Stable:
+                case ReadyToExecute:
+                    return Command.SerializerSupport.committed(attributes(), saveStatus, executeAt, Ballot.ZERO, Ballot.ZERO, Command.WaitingOn.EMPTY);
+
+                case PreApplied:
+                case Applying:
+                case Applied:
+                    return Command.SerializerSupport.executed(attributes(), saveStatus, executeAt, Ballot.ZERO, Ballot.ZERO, Command.WaitingOn.EMPTY, new Writes(txnId, executeAt, txn.keys(), new TxnWrite(Collections.emptyList(), true)), new TxnData());
+
+                case TruncatedApplyWithDeps:
+                case TruncatedApply:
+                    if (txnId.kind().awaitsOnlyDeps()) return Command.SerializerSupport.truncatedApply(attributes(), saveStatus, executeAt, null, null, txnId);
+                    else return Command.SerializerSupport.truncatedApply(attributes(), saveStatus, executeAt, null, null);
+
+                case TruncatedApplyWithOutcome:
+                    if (txnId.kind().awaitsOnlyDeps()) return Command.SerializerSupport.truncatedApply(attributes(), saveStatus, executeAt, new Writes(txnId, executeAt, txn.keys(), new TxnWrite(Collections.emptyList(), true)), new TxnData(), txnId);
+                    else return Command.SerializerSupport.truncatedApply(attributes(), saveStatus, executeAt, new Writes(txnId, executeAt, txn.keys(), new TxnWrite(Collections.emptyList(), true)), new TxnData());
+
+                case Erased:
+                case ErasedOrInvalidated:
+                case Invalidated:
+                    return Command.SerializerSupport.invalidated(txnId, Listeners.Immutable.EMPTY);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Cmd{" +
+                   "txnId=" + txnId +
+                   ", saveStatus=" + saveStatus +
+                   ", txn=" + txn +
+                   ", executeAt=" + executeAt +
+                   ", deps=" + deps +
+                   ", missing=" + missing +
+                   ", invisible=" + invisible +
+                   '}';
+        }
+    }
+
+    static class ObjectGraph
+    {
+        final Cmd[] cmds;
+        ObjectGraph(Cmd[] cmds)
+        {
+            this.cmds = cmds;
+        }
+
+        List<Command> toCommands()
+        {
+            List<Command> commands = new ArrayList<>(cmds.length);
+            for (int i = 0 ; i < cmds.length ; ++i)
+                commands.add(cmds[i].toCommand());
+            return commands;
+        }
+    }
+
+    private static ObjectGraph generateObjectGraph(int txnIdCount, Supplier<TxnId> txnIdSupplier, Supplier<SaveStatus> saveStatusSupplier, Function<TxnId, PartialTxn> txnSupplier, Function<TxnId, Timestamp> timestampSupplier, IntSupplier missingCountSupplier, RandomSource source)
+    {
+        Cmd[] cmds = new Cmd[txnIdCount];
+        for (int i = 0 ; i < txnIdCount ; ++i)
+        {
+            TxnId txnId = txnIdSupplier.get();
+            SaveStatus saveStatus = saveStatusSupplier.get();
+            Timestamp executeAt = txnId;
+            if (saveStatus.known.executeAt != ExecuteAtErased && saveStatus.known.executeAt != ExecuteAtUnknown)
+                executeAt = timestampSupplier.apply(txnId);
+
+            cmds[i] = new Cmd(txnId, txnSupplier.apply(txnId), saveStatus, executeAt);
+        }
+        Arrays.sort(cmds, Comparator.comparing(o -> o.txnId));
+        for (int i = 0 ; i < txnIdCount ; ++i)
+        {
+            if (!cmds[i].saveStatus.known.deps.hasProposedOrDecidedDeps())
+                continue;
+
+            Timestamp knownBefore = cmds[i].saveStatus.known.deps.hasCommittedOrDecidedDeps() ? cmds[i].executeAt : cmds[i].txnId;
+            int limit = SortedArrays.binarySearch(cmds, 0, cmds.length, knownBefore, (a, b) -> a.compareTo(b.txnId), FAST);
+            if (limit < 0) limit = -1 - limit;
+
+            List<TxnId> deps = cmds[i].deps;
+            List<TxnId> missing = cmds[i].missing;
+            for (int j = 0 ; j < limit ; ++j)
+                if (i != j) deps.add(cmds[j].txnId);
+
+            int missingCount = Math.min(limit - (limit > i ? 1 : 0), missingCountSupplier.getAsInt());
+            while (missingCount > 0)
+            {
+                int remove = source.nextInt(deps.size());
+                int cmdIndex = SortedArrays.binarySearch(cmds, 0, cmds.length, deps.get(remove), (a, b) -> a.compareTo(b.txnId), FAST);
+                if (!cmds[cmdIndex].saveStatus.hasBeen(Status.Committed))
+                    missing.add(deps.get(remove));
+                deps.set(remove, deps.get(deps.size() - 1));
+                deps.remove(deps.size() - 1);
+                --missingCount;
+            }
+            deps.sort(TxnId::compareTo);
+            missing.sort(TxnId::compareTo);
+        }
+
+        outer: for (int i = 0 ; i < cmds.length ; ++i)
+        {
+            if (null != InternalStatus.from(cmds[i].saveStatus))
+                continue;
+
+            for (int j = 0 ; j < i ; ++j)
+            {
+                InternalStatus status = InternalStatus.from(cmds[j].saveStatus);
+                if (status == null || !status.hasInfo) continue;
+                if (status.depsKnownBefore(cmds[j].txnId, cmds[j].executeAt).compareTo(cmds[i].txnId) > 0 && Collections.binarySearch(cmds[j].missing, cmds[i].txnId) < 0)
+                    continue outer;
+            }
+            for (int j = i + 1 ; j < cmds.length ; ++j)
+            {
+                InternalStatus status = InternalStatus.from(cmds[j].saveStatus);
+                if (status == null || !status.hasInfo) continue;
+                if (Collections.binarySearch(cmds[j].missing, cmds[i].txnId) < 0)
+                    continue outer;
+            }
+            cmds[i].invisible = true;
+            for (int j = 0 ; j < i ; ++j)
+            {
+                if (cmds[j].executeAt.compareTo(cmds[i].txnId) > 0)
+                {
+                    int remove = Collections.binarySearch(cmds[j].missing, cmds[i].txnId);
+                    if (remove >= 0) cmds[j].missing.remove(remove);
+                }
+            }
+            for (int j = i + 1 ; j < cmds.length ; ++j)
+            {
+                int remove = Collections.binarySearch(cmds[j].missing, cmds[i].txnId);
+                if (remove >= 0) cmds[j].missing.remove(remove);
+            }
+        }
+        return new ObjectGraph(cmds);
+    }
+
+    private static Function<Timestamp, TxnId> txnIdSupplier(LongUnaryOperator epochSupplier, LongUnaryOperator hlcSupplier, Supplier<Txn.Kind> kindSupplier, Supplier<Node.Id> idSupplier)
+    {
+        return min -> new TxnId(epochSupplier.applyAsLong(min == null ? 1 : min.epoch()), hlcSupplier.applyAsLong(min == null ? 1 : min.hlc() + 1), kindSupplier.get(), Routable.Domain.Key, idSupplier.get());
+    }
+
+    private static Function<Timestamp, Timestamp> timestampSupplier(LongUnaryOperator epochSupplier, LongUnaryOperator hlcSupplier, IntSupplier flagSupplier, Supplier<Node.Id> idSupplier)
+    {
+        return min -> Timestamp.fromValues(epochSupplier.applyAsLong(min == null ? 1 : min.epoch()), hlcSupplier.applyAsLong(min == null ? 1 : min.hlc() + 1), flagSupplier.getAsInt(), idSupplier.get());
+    }
+
+    private static <T extends Timestamp> Function<Timestamp, T> timestampSupplier(Set<Timestamp> unique, Function<Timestamp, T> supplier)
+    {
+        return min -> {
+            T candidate = supplier.apply(min);
+            while (!unique.add(candidate))
+            {
+                T next = supplier.apply(min);
+                if (next.equals(candidate)) min = candidate;
+                else candidate = next;
+            }
+            return candidate;
+        };
     }
 
     @Test
     public void serde()
     {
-        CommandTimeseries.CommandLoader<ByteBuffer> loader = CommandsForKeySerializer.loader;
-        qt().forAll(AccordGenerators.commands()).check(cmd -> {
-            ByteBuffer bb = loader.saveForCFK(cmd);
-            int size = bb.remaining();
+//        testOne(1821931462020409370L);
+        Random random = new Random();
+        for (int i = 0 ; i < 10000 ; ++i)
+        {
+            long seed = random.nextLong();
+            testOne(seed);
+        }
+    }
 
-            assertThat(loader.txnId(bb)).isEqualTo(cmd.txnId());
-            assertThat(bb.remaining()).describedAs("ByteBuffer was mutated").isEqualTo(size);
+    private static void testOne(long seed)
+    {
+        try
+        {
+            System.out.println(seed);
+            RandomSource source = RandomSource.wrap(new Random(seed));
 
-            assertThat(loader.executeAt(bb)).isEqualTo(cmd.executeAt());
-            assertThat(bb.remaining()).describedAs("ByteBuffer was mutated").isEqualTo(size);
+            // TODO (required): produce broader variety of distributions, including executeAt with lower HLC but higher epoch
+            final LongUnaryOperator epochSupplier; {
+                long maxEpoch = source.nextLong(1, 10);
+                epochSupplier = min -> min >= maxEpoch ? min : maxEpoch == 1 ? 1 : source.nextLong(min, maxEpoch);
+            }
+            final LongUnaryOperator hlcSupplier; {
+                long maxHlc = source.nextLong(10, 1000000);
+                hlcSupplier = min -> min >= maxHlc ? min : source.nextLong(min, maxHlc);
+            }
+            final Supplier<Node.Id> idSupplier; {
+                int maxId = source.nextInt(1, 10);
+                Int2ObjectHashMap<Node.Id> lookup = new Int2ObjectHashMap<>();
+                idSupplier = () -> lookup.computeIfAbsent(maxId == 1 ? 1 : source.nextInt(1, maxId), Node.Id::new);
+            }
+            final IntSupplier flagSupplier = () -> 0;
+            final Supplier<Txn.Kind> kindSupplier = () -> {
+                float v = source.nextFloat();
+                if (v < 0.5) return Txn.Kind.Read;
+                if (v < 0.95) return Txn.Kind.Write;
+                if (v < 0.99) return Txn.Kind.ExclusiveSyncPoint;
+                return Txn.Kind.EphemeralRead; // not actually a valid value for CFK
+            };
 
-            assertThat(loader.saveStatus(bb)).isEqualTo(cmd.saveStatus());
-            assertThat(bb.remaining()).describedAs("ByteBuffer was mutated").isEqualTo(size);
+            boolean permitMissing = source.decide(0.75f);
+            final IntSupplier missingCountSupplier; {
+                if (!permitMissing)
+                {
+                    missingCountSupplier = () -> 0;
+                }
+                else
+                {
+                    float zeroChance = source.nextFloat();
+                    int maxMissing = source.nextInt(1, 10);
+                    missingCountSupplier = () -> {
+                        float v = source.nextFloat();
+                        if (v < zeroChance) return 0;
+                        return source.nextInt(0, maxMissing);
+                    };
+                }
+            }
 
-            assertThat(loader.depsIds(bb)).isEqualTo(cmd.partialDeps() == null ? null : cmd.partialDeps().txnIds());
-            assertThat(bb.remaining()).describedAs("ByteBuffer was mutated").isEqualTo(size);
+            Choices<SaveStatus> saveStatusChoices = Choices.uniform(SaveStatus.values());
+            Supplier<SaveStatus> saveStatusSupplier = () -> {
+                SaveStatus result = saveStatusChoices.choose(source);
+                while (result == SaveStatus.TruncatedApplyWithDeps) // not a real save status
+                    result = saveStatusChoices.choose(source);
+                return result;
+            };
+
+            Set<Timestamp> uniqueTs = new TreeSet<>();
+            final Function<Timestamp, TxnId> txnIdSupplier = timestampSupplier(uniqueTs, txnIdSupplier(epochSupplier, hlcSupplier, kindSupplier, idSupplier));
+            boolean permitExecuteAt = source.decide(0.75f);
+            final Function<TxnId, Timestamp> executeAtSupplier;
+            {
+                if (!permitExecuteAt)
+                {
+                    executeAtSupplier = id -> id;
+                }
+                else
+                {
+                    Function<Timestamp, Timestamp> rawTimestampSupplier = timestampSupplier(uniqueTs, timestampSupplier(epochSupplier, hlcSupplier, flagSupplier, idSupplier));
+                    float useTxnIdChance = source.nextFloat();
+                    BooleanSupplier useTxnId = () -> source.decide(useTxnIdChance);
+                    executeAtSupplier = txnId -> useTxnId.getAsBoolean() ? txnId : rawTimestampSupplier.apply(txnId);
+                }
+            }
+
+            PartialTxn txn = createPartialTxn(0);
+            Key key = (Key) txn.keys().get(0);
+            ObjectGraph graph = generateObjectGraph(source.nextInt(0, 100), () -> txnIdSupplier.apply(null), saveStatusSupplier, ignore -> txn, executeAtSupplier, missingCountSupplier, source);
+            List<Command> commands = graph.toCommands();
+            CommandsForKey cfk = new CommandsForKey(key);
+            while (commands.size() > 0)
+            {
+                int next = source.nextInt(commands.size());
+                cfk = cfk.update(null, commands.get(next));
+                commands.set(next, commands.get(commands.size() - 1));
+                commands.remove(commands.size() - 1);
+            }
+
+            for (int i = 0, j = 0 ; j < graph.cmds.length ; ++j)
+            {
+                Cmd cmd = graph.cmds[j];
+                if (i >= cfk.size() || !cfk.txnId(i).equals(cmd.txnId))
+                {
+                    Assert.assertTrue(cmd.invisible);
+                    continue;
+                }
+                CommandsForKey.Info info = cfk.info(i);
+                InternalStatus expectStatus = InternalStatus.from(cmd.saveStatus);
+                if (expectStatus == null) expectStatus = InternalStatus.TRANSITIVELY_KNOWN;
+                if (expectStatus.hasInfo)
+                    Assert.assertEquals(cmd.executeAt, info.executeAt(cfk.txnId(i)));
+                Assert.assertEquals(expectStatus, info.status);
+                Assert.assertArrayEquals(cmd.missing.toArray(TxnId[]::new), info.missing);
+                ++i;
+            }
+
+            ByteBuffer buffer = CommandsForKeySerializer.toBytesWithoutKey(cfk);
+            CommandsForKey roundTrip = CommandsForKeySerializer.fromBytes(key, buffer);
+            Assert.assertEquals(cfk, roundTrip);
+        }
+        catch (Throwable t)
+        {
+            throw new AssertionError(seed + " seed failed", t);
+        }
+    }
+
+    @Test
+    public void test()
+    {
+        var tableGen = AccordGenerators.fromQT(CassandraGenerators.TABLE_ID_GEN);
+        var txnIdGen = AccordGens.txnIds(rs -> rs.nextLong(0, 100), rs -> rs.nextLong(100), rs -> rs.nextInt(10));
+        qt().check(rs -> {
+            TableId table = tableGen.next(rs);
+            PartitionKey pk = new PartitionKey(table, Murmur3Partitioner.instance.decorateKey(Murmur3Partitioner.LongToken.keyForToken(rs.nextLong())));
+            var redudentBefore = txnIdGen.next(rs);
+            TxnId[] ids = Gens.arrays(TxnId.class, rs0 -> {
+                TxnId next = txnIdGen.next(rs0);
+                while (next.compareTo(redudentBefore) <= 0)
+                    next = txnIdGen.next(rs0);
+                return next;
+            }).unique().ofSizeBetween(0, 10).next(rs);
+            CommandsForKey.Info[] info = new CommandsForKey.Info[ids.length];
+            for (int i = 0; i < info.length; i++)
+                info[i] = rs.pick(InternalStatus.values()).asNoInfo;
+            Arrays.sort(ids, Comparator.naturalOrder());
+            CommandsForKey expected = CommandsForKey.SerializerSupport.create(pk, redudentBefore, ids, info);
+
+            ByteBuffer buffer = CommandsForKeySerializer.toBytesWithoutKey(expected);
+            CommandsForKey roundTrip = CommandsForKeySerializer.fromBytes(pk, buffer);
+            Assert.assertEquals(expected, roundTrip);
         });
     }
+
+    @Test
+    public void thereAndBackAgain()
+    {
+        long tokenValue = -2311778975040348869L;
+        DecoratedKey key = Murmur3Partitioner.instance.decorateKey(Murmur3Partitioner.LongToken.keyForToken(tokenValue));
+        PartitionKey pk = new PartitionKey(TableId.fromString("1b255f4d-ef25-40a6-0000-000000000009"), key);
+        CommandsForKey expected = CommandsForKey.SerializerSupport.create(pk,
+                                                     TxnId.fromValues(0,0,0,0),
+                                                     new TxnId[] {TxnId.fromValues(11,34052499,2,1)},
+                                                     new CommandsForKey.Info[] { InternalStatus.PREACCEPTED.asNoInfo});
+
+        ByteBuffer buffer = CommandsForKeySerializer.toBytesWithoutKey(expected);
+        CommandsForKey roundTrip = CommandsForKeySerializer.fromBytes(pk, buffer);
+        Assert.assertEquals(expected, roundTrip);
+    }
 }