Accord: PreLoadContext must properly and consistently support ranges

patch by David Capwell; reviewed by Benedict Elliott Smith for CASSANDRA-19355

.build/build-resolver.xml

@@ -253,10 +253,6 @@
 
         <delete file="${build.lib}/netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar" failonerror="false"/>
         <delete file="${build.dir.lib}/jars/netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar" failonerror="false"/>
-
-        <copy todir="${test.lib}/jars" quiet="false">
-            <file file="${build.lib}/harry-core-0.0.2-CASSANDRA-18768.jar"/>
-       </copy>
     </target>
 
     <target name="_resolver-dist-lib_get_files">

modules/accord

@@ -1 +1 @@
-Subproject commit 3562bb3c9ce4e9eecdf65e236e968ef3ee9e0a86
+Subproject commit f78d1da27b09f89417dd29bde0529f12cd744e3d

src/java/org/apache/cassandra/config/AccordSpec.java

@@ -70,4 +70,5 @@ public class AccordSpec
      */
     public TransactionalMode default_transactional_mode = TransactionalMode.off;
     public boolean ephemeralReadEnabled = false;
+    public boolean state_cache_listener_jfr_enabled = true;
 }

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -212,6 +212,7 @@ public enum CassandraRelevantProperties
      */
     DRAIN_EXECUTOR_TIMEOUT_MS("cassandra.drain_executor_timeout_ms", convertToString(TimeUnit.MINUTES.toMillis(5))),
     DROP_OVERSIZED_READ_REPAIR_MUTATIONS("cassandra.drop_oversized_readrepair_mutations"),
+    DTEST_ACCORD_ENABLED("jvm_dtest.accord.enabled", "true"),
     DTEST_API_LOG_TOPOLOGY("cassandra.dtest.api.log.topology"),
     /** This property indicates if the code is running under the in-jvm dtest framework */
     DTEST_IS_IN_JVM_DTEST("org.apache.cassandra.dtest.is_in_jvm_dtest"),
@@ -572,7 +573,7 @@ public enum CassandraRelevantProperties
     TCM_UNSAFE_BOOT_WITH_CLUSTERMETADATA("cassandra.unsafe_boot_with_clustermetadata", null),
     TCM_USE_ATOMIC_LONG_PROCESSOR("cassandra.test.use_atomic_long_processor", "false"),
     TCM_USE_NO_OP_REPLICATOR("cassandra.test.use_no_op_replicator", "false"),
-
+    TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED("cassandra.test.accord.store.thread_checks_enabled", "true"),
     TEST_BBFAILHELPER_ENABLED("test.bbfailhelper.enabled"),
     TEST_BLOB_SHARED_SEED("cassandra.test.blob.shared.seed", "42"),
     TEST_BYTEMAN_TRANSFORMATIONS_DEBUG("cassandra.test.byteman.transformations.debug"),

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -5370,6 +5370,11 @@ public class DatabaseDescriptor
         conf.accord.shard_durability_cycle = new DurationSpec.IntSecondsBound(seconds);
     }
 
+    public static boolean getAccordStateCacheListenerJFREnabled()
+    {
+        return conf.accord.state_cache_listener_jfr_enabled;
+    }
+
     public static boolean getForceNewPreparedStatementBehaviour()
     {
         return conf.force_new_prepared_statement_behaviour;

src/java/org/apache/cassandra/index/accord/CheckpointIntervalArrayIndex.java

@@ -0,0 +1,721 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+import accord.utils.AsymmetricComparator;
+import accord.utils.CheckpointIntervalArray;
+import accord.utils.CheckpointIntervalArrayBuilder;
+import accord.utils.CheckpointIntervalArrayBuilder.Accessor;
+import accord.utils.SortedArrays;
+import org.apache.cassandra.index.accord.IndexDescriptor.IndexComponent;
+import org.apache.cassandra.io.util.ChecksumedRandomAccessReader;
+import org.apache.cassandra.io.util.ChecksumedSequentialWriter;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Throwables;
+
+import static accord.utils.CheckpointIntervalArrayBuilder.Links.LINKS;
+import static accord.utils.CheckpointIntervalArrayBuilder.Strategy.ACCURATE;
+
+//TODO (now): Add support for variable length tokens; this is needed for Ordered partitioner (which we plan to support)
+public class CheckpointIntervalArrayIndex
+{
+    private static final Accessor<Interval[], Interval, byte[]> LIST_INTERVAL_ACCESSOR = new Accessor<>()
+    {
+        @Override
+        public int size(Interval[] intervals)
+        {
+            return intervals.length;
+        }
+
+        @Override
+        public Interval get(Interval[] intervals, int index)
+        {
+            return intervals[index];
+        }
+
+        @Override
+        public byte[] start(Interval[] intervals, int index)
+        {
+            return intervals[index].start;
+        }
+
+        @Override
+        public byte[] start(Interval interval)
+        {
+            return interval.start;
+        }
+
+        @Override
+        public byte[] end(Interval[] intervals, int index)
+        {
+            return intervals[index].end;
+        }
+
+        @Override
+        public byte[] end(Interval interval)
+        {
+            return interval.end;
+        }
+
+        @Override
+        public Comparator<byte[]> keyComparator()
+        {
+            return (a, b) -> ByteArrayUtil.compareUnsigned(a, 0, b, 0, a.length);
+        }
+
+        @Override
+        public int binarySearch(Interval[] intervals, int from, int to, byte[] find, AsymmetricComparator<byte[], Interval> comparator, SortedArrays.Search op)
+        {
+            return SortedArrays.binarySearch(intervals, from, to, find, comparator, op);
+        }
+    };
+    public static final Supplier<Checksum> CHECKSUM_SUPPLIER = CRC32C::new;
+
+    //TODO (performance): rather than row structure, would column structure be better?  Sorted tokens tend to have prefix relationships
+    // so could compress the data more.  The negative here is the binary search might cost more and the scan after the random access needs end + value...
+    // Its also possible to do a hybrid structure where either start/end are column and the other one is row based...
+    //TODO (performance): store min/max values so could filter based off metadata without having to walk the tree first?  This means that the metadata
+    // doesn't need to be stored in-memory 100% of the time and only when a file "could" match.  The perf here would trade read costs for less memory.
+    //TODO (fault tolerence): right now there is no checksumming outside of the header, so a corruption in the middle
+    // could lead to weird behavior... since this structure is fixed lenght it "should" only lead to mismatches or binary
+    // search going the wrong direction...
+    //TODO (fault tolerence): maybe replace readStart/End with readRecord and extrat the value from there, this makes it so it would be trivial to add a checksum per-record.
+    // Given the migration from SAI work, we can now remove the TableId from the data (16 bytes) so a 4 byte footer wouldn't be a big cost.  We also compute the checksum on read/write
+    // right now, just ignore the value... the performance is currently better than with SAI (less overhead as we are not generic), so the checksumming costs are effectivally 0.
+    public static class SortedListWriter
+    {
+        private final int bytesPerKey, bytesPerValue;
+
+        public SortedListWriter(int bytesPerKey, int bytesPerValue)
+        {
+            this.bytesPerKey = bytesPerKey;
+            this.bytesPerValue = bytesPerValue;
+        }
+
+        public long write(ChecksumedSequentialWriter out, Interval[] sortedIntervals, Callback callback) throws IOException
+        {
+            long treeFilePointer = out.getFilePointer();
+            // write header
+            out.resetChecksum(); // reset checksum so the header is isolated
+            out.writeUnsignedVInt32(bytesPerKey);
+            out.writeUnsignedVInt32(bytesPerValue);
+            out.writeUnsignedVInt32(sortedIntervals.length);
+            out.writeInt(out.getValue32AndResetChecksum());
+
+            // write values
+            callback.preWalk(sortedIntervals);
+            int count = 0;
+            for (Interval it : sortedIntervals)
+            {
+                validate(count, it);
+
+                out.resetChecksum();
+                out.write(it.start, 0, it.start.length);
+                out.write(it.end, 0, it.end.length);
+                out.write(it.value, 0, it.value.length);
+                out.writeInt(out.getValue32());
+                callback.onWrite(count, it);
+                count++;
+            }
+            //TODO (now): don't need as this was here only for SAI.  Offset/position are the same now
+            return count == 0 ? -1 : treeFilePointer;
+        }
+
+        private void validate(int numIntervals, Interval it)
+        {
+            if (it.start.length != bytesPerKey)
+                throw new IllegalArgumentException("Interval " + numIntervals + "'s start value is size " + it.start.length + ", but expected " + bytesPerKey);
+            if (it.end.length != bytesPerKey)
+                throw new IllegalArgumentException("Interval " + numIntervals + "'s end value is size " + it.end.length + ", but expected " + bytesPerKey);
+            if (it.value.length != bytesPerValue)
+                throw new IllegalArgumentException("Interval " + numIntervals + "'s value is size " + it.value.length + ", but expected " + bytesPerValue);
+        }
+
+        public interface Callback
+        {
+            default void preWalk(Interval[] sortedIntervals) throws IOException
+            {
+            }
+
+            default void onWrite(int index, Interval interval) throws IOException
+            {
+            }
+        }
+    }
+
+    public static class SortedListReader implements Closeable
+    {
+        private final FileHandle fh;
+        private final long firstRecordOffset;
+        private final int bytesPerKey, bytesPerValue, recordSize;
+        private final int count;
+
+        public SortedListReader(FileHandle fh, long pos)
+        {
+            this.fh = fh;
+
+            try (RandomAccessReader reader = fh.createReader();
+                 ChecksumedRandomAccessReader in = new ChecksumedRandomAccessReader(reader, CHECKSUM_SUPPLIER))
+            {
+                if (pos != -1)
+                    in.seek(pos);
+
+                bytesPerKey = in.readUnsignedVInt32();
+                bytesPerValue = in.readUnsignedVInt32();
+                recordSize = bytesPerKey * 2 + bytesPerValue + Integer.BYTES;
+                count = in.readUnsignedVInt32();
+                int actualChecksum = in.getValue32AndResetChecksum();
+                int expectedChecksum = in.readInt();
+                assert actualChecksum == expectedChecksum;
+                firstRecordOffset = reader.getFilePointer();
+            }
+            catch (Throwable t)
+            {
+                FileUtils.closeQuietly(fh);
+                throw Throwables.unchecked(t);
+            }
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            FileUtils.closeQuietly(fh);
+        }
+
+        public enum SeekReason
+        {BINARY_SEARCH, GET, SCAN}
+
+        private boolean maybeSeek(ChecksumedRandomAccessReader indexInput, Stats stats, SeekReason reason, long target) throws IOException
+        {
+            if (indexInput.getFilePointer() != target)
+            {
+                indexInput.seek(target);
+                switch (reason)
+                {
+                    case SCAN:
+                        stats.seekForScan++;
+                        break;
+                    case GET:
+                        stats.seekForGet++;
+                        break;
+                    case BINARY_SEARCH:
+                        stats.seekForBinarySearch++;
+                        break;
+                    default:
+                        throw new IllegalArgumentException("Unknown reason: " + reason);
+                }
+                return true;
+            }
+            return false;
+        }
+
+        public byte[] getRecord(ChecksumedRandomAccessReader indexInput, Stats stats, SeekReason reason, byte[] recordBuffer, int pos) throws IOException
+        {
+            maybeSeek(indexInput, stats, reason, fileOffsetStart(pos));
+            return getCurrentRecord(indexInput, stats, recordBuffer);
+        }
+
+        public byte[] getCurrentRecord(ChecksumedRandomAccessReader indexInput, Stats stats, byte[] recordBuffer) throws IOException
+        {
+            stats.bytesRead += recordBuffer.length + Integer.BYTES;
+            indexInput.resetChecksum();
+            indexInput.readFully(recordBuffer, 0, recordBuffer.length);
+            int actualChecksum = indexInput.getValue32();
+            int expectedChecksum = indexInput.readInt();
+            assert actualChecksum == expectedChecksum;
+            return recordBuffer;
+        }
+
+        public byte[] readStart(ChecksumedRandomAccessReader indexInput, Stats stats, byte[] recordBuffer, byte[] keyBuffer, int pos) throws IOException
+        {
+            getRecord(indexInput, stats, SeekReason.GET, recordBuffer, pos);
+            copyStart(recordBuffer, keyBuffer);
+            return keyBuffer;
+        }
+
+        public byte[] readEnd(ChecksumedRandomAccessReader indexInput, Stats stats, byte[] recordBuffer, byte[] keyBuffer, int pos) throws IOException
+        {
+            getRecord(indexInput, stats, SeekReason.GET, recordBuffer, pos);
+            copyEnd(recordBuffer, keyBuffer);
+            return keyBuffer;
+        }
+
+        public byte[] copyStart(byte[] recordBuffer, byte[] keyBuffer)
+        {
+            System.arraycopy(recordBuffer, 0, keyBuffer, 0, keyBuffer.length);
+            return keyBuffer;
+        }
+
+        public byte[] copyEnd(byte[] recordBuffer, byte[] keyBuffer)
+        {
+            System.arraycopy(recordBuffer, bytesPerKey, keyBuffer, 0, bytesPerKey);
+            return keyBuffer;
+        }
+
+        public int binarySearch(ChecksumedRandomAccessReader indexInput, Stats stats, byte[] recordBuffer, int from, int to, byte[] find, AsymmetricComparator<byte[], byte[]> comparator, SortedArrays.Search op) throws IOException
+        {
+            int found = -1;
+            while (from < to)
+            {
+                int i = (from + to) >>> 1;
+                int c = comparator.compare(find, getRecord(indexInput, stats, SeekReason.BINARY_SEARCH, recordBuffer, i));
+                if (c < 0)
+                {
+                    to = i;
+                }
+                else if (c > 0)
+                {
+                    from = i + 1;
+                }
+                else
+                {
+                    switch (op)
+                    {
+                        default:
+                            throw new IllegalStateException("Unknown search operation: " + op);
+                        case FAST:
+                            return i;
+
+                        case CEIL:
+                            to = found = i;
+                            break;
+
+                        case FLOOR:
+                            found = i;
+                            from = i + 1;
+                    }
+                }
+            }
+            // return -(low + 1);  // key not found.
+            return found >= 0 ? found : -1 - to;
+        }
+
+        public Interval copyTo(byte[] record, Interval buffer)
+        {
+            buffer.start = Arrays.copyOfRange(record, 0, bytesPerKey);
+            buffer.end = Arrays.copyOfRange(record, bytesPerKey, bytesPerKey * 2);
+            buffer.value = Arrays.copyOfRange(record, bytesPerKey * 2, record.length);
+            return buffer;
+        }
+
+        private long fileOffsetStart(int offset)
+        {
+            if (offset >= count)
+                throw new IndexOutOfBoundsException("Start is from (0, " + count + "]; attempted to access " + offset);
+            return firstRecordOffset + (offset * recordSize);
+        }
+
+        private long fileOffsetEnd(int offset)
+        {
+            if (offset >= count)
+                throw new IndexOutOfBoundsException("Start is from (0, " + count + "]; attempted to access " + offset);
+            return firstRecordOffset + (offset * recordSize) + bytesPerKey;
+        }
+
+        private long fileOffsetValue(int offset)
+        {
+            if (offset >= count)
+                throw new IndexOutOfBoundsException("Start is from (0, " + count + "]; attempted to access " + offset);
+            return firstRecordOffset + (offset * recordSize) + bytesPerKey * 2;
+        }
+    }
+
+    public static class CheckpointWriter implements SortedListWriter.Callback, Closeable
+    {
+        private final ChecksumedSequentialWriter out;
+        private final long offset;
+        private final long position; //TODO (now): don't need as this was here only for SAI.  Offset/position are the same now
+        private long length = -1;
+
+        public CheckpointWriter(ChecksumedSequentialWriter out)
+        {
+            this.out = out;
+            this.offset = position = out.getFilePointer();
+        }
+
+        @Override
+        public void preWalk(Interval[] sortedIntervals) throws IOException
+        {
+            class Checkpoints
+            {
+                final int[] bounds, headers, lists;
+                final int maxScanAndCheckpointMatches;
+
+                Checkpoints(int[] bounds, int[] headers, int[] lists, int maxScanAndCheckpointMatches)
+                {
+                    this.bounds = bounds;
+                    this.headers = headers;
+                    this.lists = lists;
+                    this.maxScanAndCheckpointMatches = maxScanAndCheckpointMatches;
+                }
+            }
+            Checkpoints c = new CheckpointIntervalArrayBuilder<>(LIST_INTERVAL_ACCESSOR, sortedIntervals, ACCURATE, LINKS).build((ignore, bounds, headers, lists, max) -> new Checkpoints(bounds, headers, lists, max));
+            out.resetChecksum(); // reset checksum so it only covers this metadata
+            out.writeUnsignedVInt32(c.maxScanAndCheckpointMatches);
+            write(c.bounds);
+            write(c.headers);
+            write(c.lists);
+            out.writeInt(out.getValue32AndResetChecksum());
+        }
+
+        private void write(int[] array) throws IOException
+        {
+            out.writeUnsignedVInt32(array.length);
+            for (int i = 0; i < array.length; i++)
+                out.writeVInt32(array[i]);
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            length = out.getFilePointer() - offset;
+            out.close();
+        }
+    }
+
+    //TODO (performance): the current format assumes random list access is cheap, which isn't true for a disk index.
+    // This format was chosen as a place holder for now so we don't drift from the in-memory logic; in the original paper
+    // a new sorted list is used for each checkpoint, which then makes the access a sequential scan rather than random access.
+    public static class CheckpointReader implements Closeable
+    {
+        private final FileHandle fh;
+        private final int[] bounds, headers, lists;
+        private final int maxScanAndCheckpointMatches;
+
+        public CheckpointReader(FileHandle fh, long pos)
+        {
+            this.fh = fh;
+            try (RandomAccessReader reader = fh.createReader();
+                 ChecksumedRandomAccessReader input = new ChecksumedRandomAccessReader(reader, CHECKSUM_SUPPLIER))
+            {
+                if (pos != -1)
+                    input.seek(pos);
+
+                input.resetChecksum(); // reset checksum so it only covers this metadata
+                maxScanAndCheckpointMatches = input.readUnsignedVInt32();
+                bounds = readArray(input);
+                headers = readArray(input);
+                lists = readArray(input);
+                int actualChecksum = input.getValue32AndResetChecksum();
+                int expectedChecksum = input.readInt();
+                assert actualChecksum == expectedChecksum;
+            }
+            catch (Throwable t)
+            {
+                FileUtils.closeQuietly(fh);
+                throw Throwables.unchecked(t);
+            }
+        }
+
+        private static int[] readArray(ChecksumedRandomAccessReader input) throws IOException
+        {
+            int size = input.readUnsignedVInt32();
+            int[] array = new int[size];
+            for (int i = 0; i < size; i++)
+                array[i] = input.readVInt32();
+            return array;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            FileUtils.closeQuietly(fh);
+        }
+    }
+
+    public static class SegmentWriter
+    {
+        private final IndexDescriptor id;
+        private final SortedListWriter writer;
+
+        public SegmentWriter(IndexDescriptor id, int bytesPerKey, int bytesPerValue)
+        {
+            this.id = id;
+            this.writer = new SortedListWriter(bytesPerKey, bytesPerValue);
+        }
+
+        public EnumMap<IndexComponent, Segment.ComponentMetadata> write(Interval[] sortedIntervals) throws IOException
+        {
+            EnumMap<IndexComponent, Segment.ComponentMetadata> metas = new EnumMap<>(IndexComponent.class);
+            try (ChecksumedSequentialWriter treeOutput = ChecksumedSequentialWriter.open(id.fileFor(IndexComponent.CINTIA_SORTED_LIST), true, CHECKSUM_SUPPLIER);
+                 CheckpointWriter checkpointWriter = new CheckpointWriter(ChecksumedSequentialWriter.open(id.fileFor(IndexComponent.CINTIA_CHECKPOINTS), true, CHECKSUM_SUPPLIER)))
+            {
+                // The SSTable component file is opened in append mode, so our offset is the current file pointer.
+                long sortedOffset = treeOutput.getFilePointer();
+                long sortedPosition = writer.write(treeOutput, sortedIntervals, checkpointWriter);
+
+                // If the treePosition is less than 0 then we didn't write any values out and the index is empty
+                if (sortedPosition < 0)
+                    return metas;
+                //TODO (now): currently does SAI header so offset isn't correct here and need position
+                metas.put(IndexComponent.CINTIA_SORTED_LIST, new Segment.ComponentMetadata(sortedPosition, treeOutput.getFilePointer()));
+                metas.put(IndexComponent.CINTIA_CHECKPOINTS, new Segment.ComponentMetadata(checkpointWriter.position, checkpointWriter.out.getFilePointer()));
+            }
+            return metas;
+        }
+    }
+
+    public static class Stats
+    {
+        int seekForGet, seekForBinarySearch, seekForScan;
+        long durationNs, bytesRead, matches;
+
+        @Override
+        public String toString()
+        {
+            return "Stats{" +
+                   "seeks={Get=" + seekForGet +
+                   ", BinarySearch=" + seekForBinarySearch +
+                   ", Scan=" + seekForScan +
+                   "}, bytesRead=" + bytesRead +
+                   ", matches=" + matches +
+                   ", duration_micro=" + TimeUnit.NANOSECONDS.toMicros(durationNs) +
+                   '}';
+        }
+    }
+
+    public static class SegmentSearcher implements Closeable
+    {
+        private final SortedListReader reader;
+        private final CheckpointReader checkpoints;
+
+        public SegmentSearcher(FileHandle sortedListFile,
+                               long sortedListPosition,
+                               FileHandle checkpointFile,
+                               long checkpointPosition)
+        {
+            this.reader = new SortedListReader(sortedListFile, sortedListPosition);
+            this.checkpoints = new CheckpointReader(checkpointFile, checkpointPosition);
+        }
+
+        public Stats intersects(byte[] start, byte[] end, Consumer<Interval> callback) throws IOException
+        {
+            byte[] keyBuffer = new byte[reader.bytesPerKey];
+            byte[] recordBuffer = new byte[reader.recordSize - Integer.BYTES];
+            Stats stats = new Stats();
+            long startNanos = Clock.Global.nanoTime();
+            try (ChecksumedRandomAccessReader indexInput = new ChecksumedRandomAccessReader(reader.fh.createReader(), CHECKSUM_SUPPLIER))
+            {
+                Interval buffer = new Interval();
+                Accessor<ChecksumedRandomAccessReader, byte[], byte[]> accessor = new Accessor<>()
+                {
+                    @Override
+                    public int size(ChecksumedRandomAccessReader indexInput)
+                    {
+                        return reader.count;
+                    }
+
+                    @Override
+                    public byte[] get(ChecksumedRandomAccessReader indexInput, int index)
+                    {
+                        try
+                        {
+                            return reader.getRecord(indexInput, stats, SortedListReader.SeekReason.GET, recordBuffer, index);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
+
+                    @Override
+                    public byte[] start(ChecksumedRandomAccessReader indexInput, int index)
+                    {
+                        try
+                        {
+                            return reader.readStart(indexInput, stats, recordBuffer, keyBuffer, index);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
+
+                    @Override
+                    public byte[] start(byte[] bytes)
+                    {
+                        return reader.copyStart(bytes, keyBuffer);
+                    }
+
+                    @Override
+                    public byte[] end(ChecksumedRandomAccessReader indexInput, int index)
+                    {
+                        try
+                        {
+                            return reader.readEnd(indexInput, stats, recordBuffer, keyBuffer, index);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
+
+                    @Override
+                    public byte[] end(byte[] bytes)
+                    {
+                        return reader.copyEnd(bytes, keyBuffer);
+                    }
+
+                    @Override
+                    public Comparator<byte[]> keyComparator()
+                    {
+                        return (a, b) -> ByteArrayUtil.compareUnsigned(a, 0, b, 0, a.length);
+                    }
+
+                    @Override
+                    public int binarySearch(ChecksumedRandomAccessReader indexInput, int from, int to, byte[] find, AsymmetricComparator<byte[], byte[]> comparator, SortedArrays.Search op)
+                    {
+                        try
+                        {
+                            return reader.binarySearch(indexInput, stats, recordBuffer, from, to, find, comparator, op);
+                        }
+                        catch (IOException e)
+                        {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
+                };
+                CheckpointIntervalArray<ChecksumedRandomAccessReader, byte[], byte[]> searcher = new CheckpointIntervalArray<>(accessor, indexInput, checkpoints.bounds, checkpoints.headers, checkpoints.lists, checkpoints.maxScanAndCheckpointMatches);
+
+                searcher.forEach(start, end, (i1, i2, i3, i4, index) -> {
+                    stats.matches++;
+                    callback.accept(reader.copyTo(accessor.get(indexInput, index), buffer));
+                }, (i1, i2, i3, i4, startIdx, endIdx) -> {
+                    try
+                    {
+                        reader.maybeSeek(indexInput, stats, SortedListReader.SeekReason.SCAN, reader.fileOffsetStart(startIdx));
+                        for (int i = startIdx; i < endIdx; i++)
+                        {
+                            stats.matches++;
+                            reader.getCurrentRecord(indexInput, stats, recordBuffer);
+                            callback.accept(reader.copyTo(recordBuffer, buffer));
+                        }
+                    }
+                    catch (IOException e)
+                    {
+                        throw new UncheckedIOException(e);
+                    }
+                }, 0, 0, 0, 0, 0);
+            }
+            finally
+            {
+                stats.durationNs = Clock.Global.nanoTime() - startNanos;
+            }
+            return stats;
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(checkpoints);
+            FileUtils.closeQuietly(reader);
+        }
+    }
+
+    public static class Interval implements Comparable<Interval>
+    {
+        public byte[] start, end, value; // mutable to avoid allocating Interval for every element
+
+        public Interval()
+        {
+        }
+
+        public Interval(byte[] start, byte[] end, byte[] value)
+        {
+            this.start = start;
+            this.end = end;
+            this.value = value;
+        }
+
+        public Interval(Interval other)
+        {
+            this.start = other.start;
+            this.end = other.end;
+            this.value = other.value;
+        }
+
+        @Override
+        public int compareTo(Interval b)
+        {
+            int rc = compareStart(b);
+            if (rc == 0)
+                rc = ByteArrayUtil.compareUnsigned(value, 0, b.value, 0, value.length);
+            return rc;
+        }
+
+        public int compareStart(Interval b)
+        {
+            int rc = ByteArrayUtil.compareUnsigned(start, 0, b.start, 0, start.length);
+            if (rc == 0)
+                rc = ByteArrayUtil.compareUnsigned(end, 0, b.end, 0, end.length);
+            return rc;
+        }
+
+        public int compareEnd(Interval b)
+        {
+            int rc = ByteArrayUtil.compareUnsigned(end, 0, b.end, 0, end.length);
+            if (rc == 0)
+                rc = ByteArrayUtil.compareUnsigned(start, 0, b.start, 0, start.length);
+            return rc;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Interval interval = (Interval) o;
+            return Arrays.equals(start, interval.start) && Arrays.equals(end, interval.end) && Arrays.equals(value, interval.value);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            int result = Arrays.hashCode(start);
+            result = 31 * result + Arrays.hashCode(end);
+            result = 31 * result + Arrays.hashCode(value);
+            return result;
+        }
+
+        public boolean intersects(byte[] start, byte[] end)
+        {
+            if (ByteArrayUtil.compareUnsigned(this.start, end) >= 0)
+                return false;
+            if (ByteArrayUtil.compareUnsigned(this.end, start) <= 0)
+                return false;
+            return true;
+        }
+    }
+}

src/java/org/apache/cassandra/index/accord/Group.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.Objects;
+
+import org.apache.cassandra.schema.TableId;
+
+public class Group implements Comparable<Group>
+{
+    public final int storeId;
+    public final TableId tableId;
+
+    public Group(int storeId, TableId tableId)
+    {
+        this.storeId = storeId;
+        this.tableId = tableId;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Group group = (Group) o;
+        return storeId == group.storeId && Objects.equals(tableId, group.tableId);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(storeId, tableId);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "Group{" +
+               "storeId=" + storeId +
+               ", tableId=" + tableId +
+               '}';
+    }
+
+    @Override
+    public int compareTo(Group o)
+    {
+        int rc = Integer.compare(storeId, o.storeId);
+        if (rc == 0)
+            rc = tableId.compareTo(o.tableId);
+        return rc;
+    }
+}

src/java/org/apache/cassandra/index/accord/IndexDescriptor.java

@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.Collection;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+
+public class IndexDescriptor
+{
+    public enum Version
+    {
+        v1("aa", c -> defaultFileNameFormat(c, "aa"));
+
+        public static final Version CURRENT = v1;
+
+        public final String versionString;
+        public final Function<IndexComponent, String> fileNameFormatter;
+
+        Version(String versionString, Function<IndexComponent, String> fileNameFormatter)
+        {
+            this.versionString = versionString;
+            this.fileNameFormatter = fileNameFormatter;
+        }
+
+
+        private static String defaultFileNameFormat(IndexComponent indexComponent, String version)
+        {
+            StringBuilder sb = new StringBuilder();
+
+            sb.append(IndexComponent.DESCRIPTOR).append(IndexComponent.SEPARATOR)
+              .append(version).append(IndexComponent.SEPARATOR)
+              .append(indexComponent.name).append(Descriptor.EXTENSION);
+
+            return sb.toString();
+        }
+    }
+
+    public enum IndexComponent
+    {
+        CINTIA_SORTED_LIST("CintiaSortedList", (byte) 1),
+        CINTIA_CHECKPOINTS("CintiaCheckpoints", (byte) 2),
+        SEGMENT("Segement", (byte) 3),
+        METADATA("Metadata", (byte) 4);
+
+        public static final String DESCRIPTOR = "ACCORD";
+        public static final String SEPARATOR = "+";
+
+        public final String name;
+        public final Component.Type type;
+        public final byte value;
+
+        IndexComponent(String name, byte value)
+        {
+            this.name = name;
+            this.type = componentType(name);
+            this.value = value;
+        }
+
+        private static Component.Type componentType(String name)
+        {
+            String componentName = DESCRIPTOR + SEPARATOR + name;
+            String repr = Pattern.quote(DESCRIPTOR + SEPARATOR)
+                          + ".*"
+                          + Pattern.quote(SEPARATOR + name + ".db");
+            return Component.Type.create(componentName, repr, true, null);
+        }
+
+        public static IndexComponent fromByte(byte b)
+        {
+            switch (b)
+            {
+                case 1: return CINTIA_SORTED_LIST;
+                case 2: return CINTIA_CHECKPOINTS;
+                case 3: return SEGMENT;
+                case 4: return METADATA;
+                default:throw new IllegalArgumentException("Unknow byte: " + b);
+            }
+        }
+    }
+
+    public final Version version;
+    public final Descriptor sstableDescriptor;
+    public final IPartitioner partitioner;
+    public final ClusteringComparator clusteringComparator;
+
+    public IndexDescriptor(Version version, Descriptor sstableDescriptor, IPartitioner partitioner, ClusteringComparator clusteringComparator)
+    {
+        this.version = version;
+        this.sstableDescriptor = sstableDescriptor;
+        this.partitioner = partitioner;
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    public static IndexDescriptor create(SSTableReader sstable)
+    {
+        for (Version version : Version.values())
+        {
+            IndexDescriptor id = new IndexDescriptor(version, sstable.descriptor, sstable.getPartitioner(), sstable.metadata().comparator);
+            if (id.isIndexBuildComplete())
+                return id;
+        }
+        return new IndexDescriptor(Version.CURRENT, sstable.descriptor, sstable.getPartitioner(), sstable.metadata().comparator);
+    }
+
+    public static IndexDescriptor create(Descriptor descriptor, IPartitioner partitioner, ClusteringComparator comparator)
+    {
+        return new IndexDescriptor(Version.CURRENT, descriptor, partitioner, comparator);
+    }
+
+    public boolean isIndexBuildComplete()
+    {
+        return hasComponent(IndexComponent.METADATA);
+    }
+
+    public boolean hasComponent(IndexComponent indexComponent)
+    {
+        return fileFor(indexComponent).exists();
+    }
+
+    public File fileFor(IndexComponent indexComponent)
+    {
+        Component c = indexComponent.type.createComponent(version.fileNameFormatter.apply(indexComponent));
+        return sstableDescriptor.fileFor(c);
+    }
+
+    public void deleteIndex()
+    {
+        Stream.of(IndexComponent.values()).map(this::fileFor).forEach(File::deleteIfExists);
+    }
+
+    public Collection<Component> getLiveSSTableComponents()
+    {
+        return Stream.of(IndexComponent.values())
+                     .map(c -> c.type.createComponent(version.fileNameFormatter.apply(c)))
+                     .filter(c -> sstableDescriptor.fileFor(c).exists())
+                     .collect(Collectors.toList());
+    }
+
+    public Collection<IndexDescriptor.IndexComponent> getLiveComponents()
+    {
+        return Stream.of(IndexComponent.values())
+                     .filter(c -> fileFor(c).exists())
+                     .collect(Collectors.toList());
+    }
+}

src/java/org/apache/cassandra/index/accord/IndexMetrics.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+// Stolen from org.apache.cassandra.index.sai.metrics.AbstractMetrics
+public class IndexMetrics
+{
+    private static final String TYPE = "RouteIndex";
+    private static final String SCOPE = "IndexMetrics";
+
+    private final List<CassandraMetricsRegistry.MetricName> tracked = new ArrayList<>();
+
+    private final String ks;
+    private final String table;
+    private final String indexName;
+    public final Timer memtableIndexWriteLatency;
+
+    public IndexMetrics(RouteIndex index)
+    {
+        this.ks = index.baseCfs().getKeyspaceName();
+        this.table = index.baseCfs().name;
+        this.indexName = index.getIndexMetadata().name;
+        memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency"));
+    }
+
+    public void release()
+    {
+        tracked.forEach(Metrics::remove);
+        tracked.clear();
+    }
+
+    private CassandraMetricsRegistry.MetricName createMetricName(String name)
+    {
+        String metricScope = ks + '.' + table;
+        if (indexName != null)
+        {
+            metricScope += '.' + indexName;
+        }
+        metricScope += '.' + SCOPE + '.' + name;
+
+        CassandraMetricsRegistry.MetricName metricName = new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME,
+                                                                                                 TYPE, name, metricScope, createMBeanName(name, SCOPE));
+        tracked.add(metricName);
+        return metricName;
+    }
+
+    private String createMBeanName(String name, String scope)
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append(DefaultNameFactory.GROUP_NAME);
+        builder.append(":type=").append(TYPE);
+        builder.append(',').append("keyspace=").append(ks);
+        builder.append(',').append("table=").append(table);
+        if (indexName != null)
+            builder.append(',').append("index=").append(indexName);
+        builder.append(',').append("scope=").append(scope);
+        builder.append(',').append("name=").append(name);
+        return builder.toString();
+    }
+}

src/java/org/apache/cassandra/index/accord/MemtableIndex.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.concurrent.atomic.LongAdder;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.schema.TableId;
+
+public class MemtableIndex
+{
+    private final RangeMemoryIndex memoryIndex = new RangeMemoryIndex();
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder estimatedMemoryUsed = new LongAdder();
+
+    public long writeCount()
+    {
+        return writeCount.sum();
+    }
+
+    public long estimatedMemoryUsed()
+    {
+        return estimatedMemoryUsed.sum();
+    }
+
+    public boolean isEmpty()
+    {
+        return memoryIndex.isEmpty();
+    }
+
+    public long index(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        if (value == null || value.remaining() == 0)
+            return 0;
+        long size = memoryIndex.add(key, clustering, value);
+        writeCount.increment();
+        estimatedMemoryUsed.add(size);
+        return size;
+    }
+
+    public Segment write(IndexDescriptor id) throws IOException
+    {
+        return memoryIndex.write(id);
+    }
+
+    public Collection<ByteBuffer> search(int storeId, TableId tableId, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive)
+    {
+        return memoryIndex.search(storeId, tableId, start, startInclusive, end, endInclusive);
+    }
+}

src/java/org/apache/cassandra/index/accord/MemtableIndexManager.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.nio.ByteBuffer;
+import java.util.NavigableSet;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.TableId;
+
+public interface MemtableIndexManager
+{
+    long index(DecoratedKey key, Row row, Memtable mt);
+
+    MemtableIndex getPendingMemtableIndex(LifecycleNewTracker tracker);
+
+    void discardMemtable(Memtable memtable);
+
+    void renewMemtable(Memtable renewed);
+
+    NavigableSet<ByteBuffer> search(int storeId, TableId tableId, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive);
+}

src/java/org/apache/cassandra/index/accord/OrderedRouteSerializer.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource;
+
+public class OrderedRouteSerializer
+{
+    private static final AccordRoutingKeyByteSource.FixedLength SERIALIZER = AccordRoutingKeyByteSource.fixedLength(DatabaseDescriptor.getPartitioner());
+
+    public static ByteBuffer serializeRoutingKey(AccordRoutingKey key)
+    {
+        return ByteBuffer.wrap(SERIALIZER.serialize(key));
+    }
+
+    public static byte[] serializeRoutingKeyNoTable(AccordRoutingKey key)
+    {
+        return SERIALIZER.serializeNoTable(key);
+    }
+
+    public static byte[] unwrap(AccordRoutingKey key)
+    {
+        return SERIALIZER.serialize(key);
+    }
+
+    public static AccordRoutingKey deserializeRoutingKey(ByteBuffer bb)
+    {
+        try
+        {
+            return SERIALIZER.fromComparableBytes(ByteBufferAccessor.instance, bb);
+        }
+        catch (IOException e)
+        {
+            throw new UnsupportedOperationException(e);
+        }
+    }
+}

src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import javax.annotation.concurrent.GuardedBy;
+
+import accord.primitives.Routable;
+import accord.primitives.Route;
+import accord.primitives.Unseekable;
+import org.apache.cassandra.cache.IMeasurableMemory;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.RTree;
+import org.apache.cassandra.utils.RangeTree;
+
+public class RangeMemoryIndex
+{
+    @GuardedBy("this")
+    private final Map<Group, RangeTree<byte[], Range, DecoratedKey>> map = new HashMap<>();
+    @GuardedBy("this")
+    private final Map<Group, Metadata> groupMetadata = new HashMap<>();
+
+    private static class Metadata
+    {
+        public byte[] minTerm, maxTerm;
+    }
+
+    private static RangeTree<byte[], Range, DecoratedKey> createRangeTree()
+    {
+        return new RTree<>((a, b) -> ByteArrayUtil.compareUnsigned(a, 0, b, 0, a.length), new RangeTree.Accessor<>()
+        {
+            @Override
+            public byte[] start(Range range)
+            {
+                return range.start;
+            }
+
+            @Override
+            public byte[] end(Range range)
+            {
+                return range.end;
+            }
+
+            @Override
+            public boolean contains(byte[] start, byte[] end, byte[] bytes)
+            {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public boolean intersects(Range range, byte[] start, byte[] end)
+            {
+                return range.intersects(start, end);
+            }
+
+            @Override
+            public boolean intersects(Range left, Range right)
+            {
+                return left.intersects(right.start, right.end);
+            }
+        });
+    }
+
+    public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        Route<?> route;
+        try
+        {
+            route = AccordKeyspace.deserializeRouteOrNull(value);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+
+        return add(key, route);
+    }
+
+
+    public synchronized long add(DecoratedKey key, Route<?> route)
+    {
+        if (route.domain() != Routable.Domain.Range)
+            return 0;
+        long sum = 0;
+        for (Unseekable keyOrRange : route)
+            sum += add(key, keyOrRange);
+        return sum;
+    }
+    
+    protected long add(DecoratedKey key, Unseekable keyOrRange)
+    {
+        if (keyOrRange.domain() != Routable.Domain.Range)
+            throw new IllegalArgumentException("Unexpected domain: " + keyOrRange.domain());
+        TokenRange ts = (TokenRange) keyOrRange;
+
+        int storeId = AccordKeyspace.CommandRows.getStoreId(key);
+        TableId tableId = ts.table();
+        Group group = new Group(storeId, tableId);
+        byte[] start = OrderedRouteSerializer.serializeRoutingKeyNoTable((AccordRoutingKey) ts.start());
+        byte[] end = OrderedRouteSerializer.serializeRoutingKeyNoTable((AccordRoutingKey) ts.end());
+        Range range = new Range(start, end);
+        map.computeIfAbsent(group, ignore -> createRangeTree()).add(range, key);
+        Metadata metadata = groupMetadata.computeIfAbsent(group, ignore -> new Metadata());
+
+        metadata.minTerm = metadata.minTerm == null ? start : ByteArrayUtil.compareUnsigned(metadata.minTerm, 0, start, 0, metadata.minTerm.length) > 0 ? start : metadata.minTerm;
+        metadata.maxTerm = metadata.maxTerm == null ? end : ByteArrayUtil.compareUnsigned(metadata.maxTerm, 0, end, 0, metadata.maxTerm.length) < 0 ? end : metadata.maxTerm;
+        return TableId.EMPTY_SIZE + range.unsharedHeapSize();
+    }
+
+    public NavigableSet<ByteBuffer> search(int storeId, TableId tableId, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive)
+    {
+        RangeTree<byte[], Range, DecoratedKey> rangesToPks = map.get(new Group(storeId, tableId));
+        if (rangesToPks == null || rangesToPks.isEmpty())
+            return Collections.emptyNavigableSet();
+        TreeMap<Range, Set<DecoratedKey>> matches = search(rangesToPks, start, end);
+        if (matches.isEmpty())
+            return Collections.emptyNavigableSet();
+        TreeSet<ByteBuffer> pks = new TreeSet<>();
+        matches.values().forEach(s -> s.forEach(d -> pks.add(d.getKey())));
+        return pks;
+    }
+
+    private TreeMap<Range, Set<DecoratedKey>> search(RangeTree<byte[], Range, DecoratedKey> tokensToPks, byte[] start, byte[] end)
+    {
+
+        TreeMap<Range, Set<DecoratedKey>> matches = new TreeMap<>();
+        tokensToPks.search(new Range(start, end), e -> matches.computeIfAbsent(e.getKey(), ignore -> new HashSet<>()).add(e.getValue()));
+        return matches;
+    }
+
+    public synchronized boolean isEmpty()
+    {
+        return map.isEmpty();
+    }
+
+    public Segment write(IndexDescriptor id) throws IOException
+    {
+        if (map.isEmpty())
+            throw new AssertionError("Unable to write empty index");
+        Map<Group, Segment.Metadata> output = new HashMap<>();
+
+        List<Group> groups = new ArrayList<>(map.keySet());
+        groups.sort(Comparator.naturalOrder());
+
+        for (Group group : groups)
+        {
+            RangeTree<byte[], Range, DecoratedKey> submap = map.get(group);
+            if (submap.isEmpty()) // is this possible?  put here for safty so list is never empty
+                continue;
+            Metadata metadata = groupMetadata.get(group);
+
+            //TODO (performance): if the RangeTree can return the data in sorted order, then this local can become faster
+            // Right now the code is based off RTree, which is undefined order, so we must iterate then sort; in testing this is a good chunk of the time of this method
+            List<CheckpointIntervalArrayIndex.Interval> list = submap.stream()
+                                                                     .map(e -> new CheckpointIntervalArrayIndex.Interval(e.getKey().start, e.getKey().end, ByteBufferUtil.getArray(e.getValue().getKey())))
+                                                                     .sorted(Comparator.naturalOrder())
+                                                                     .collect(Collectors.toList());
+
+            CheckpointIntervalArrayIndex.SegmentWriter writer = new CheckpointIntervalArrayIndex.SegmentWriter(id, list.get(0).start.length, list.get(0).value.length);
+            EnumMap<IndexDescriptor.IndexComponent, Segment.ComponentMetadata> meta = writer.write(list.toArray(CheckpointIntervalArrayIndex.Interval[]::new));
+            if (meta.isEmpty()) // don't include empty segments
+                continue;
+            output.put(group, new Segment.Metadata(meta, metadata.minTerm, metadata.maxTerm));
+        }
+
+        return new Segment(output);
+    }
+
+    private static class Range implements Comparable<Range>, IMeasurableMemory
+    {
+        private static final long EMPTY_SIZE = ObjectSizes.measure(new Range(null, null));
+
+        private final byte[] start, end;
+
+        private Range(byte[] start, byte[] end)
+        {
+            this.start = start;
+            this.end = end;
+        }
+
+        @Override
+        public int compareTo(Range other)
+        {
+            int rc = ByteArrayUtil.compareUnsigned(start, 0, other.start, 0, start.length);
+            if (rc == 0)
+                rc = ByteArrayUtil.compareUnsigned(end, 0, other.end, 0, end.length);
+            return rc;
+        }
+
+        @Override
+        public long unsharedHeapSize()
+        {
+            return EMPTY_SIZE + ObjectSizes.sizeOfArray(start) * 2;
+        }
+
+        public boolean intersects(byte[] start, byte[] end)
+        {
+            if (ByteArrayUtil.compareUnsigned(this.start, 0, end, 0, end.length) >= 0)
+                return false;
+            if (ByteArrayUtil.compareUnsigned(this.end, 0, start, 0, start.length) <= 0)
+                return false;
+            return true;
+        }
+    }
+}

src/java/org/apache/cassandra/index/accord/RouteIndex.java

@@ -0,0 +1,604 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.CassandraWriteContext;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.BTreeRow;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
+
+public class RouteIndex implements Index, INotificationConsumer
+{
+    public enum RegisterStatus
+    {PENDING, REGISTERED, UNREGISTERED}
+
+    private static final Logger logger = LoggerFactory.getLogger(RouteIndex.class);
+
+    private static final Component.Type type = Component.Type.createSingleton("AccordRoute", "AccordRoute.*.db", true, null);
+
+    private final ColumnFamilyStore baseCfs;
+    private final ColumnMetadata column;
+    private final IndexMetadata indexMetadata;
+    private final IndexMetrics indexMetrics;
+    private final MemtableIndexManager memtableIndexManager;
+    private final SSTableManager sstableManager;
+    // Tracks whether we've started the index build on initialization.
+    private volatile boolean initBuildStarted = false;
+    private volatile RegisterStatus registerStatus = RegisterStatus.PENDING;
+
+    public RouteIndex(ColumnFamilyStore baseCfs, IndexMetadata indexMetadata)
+    {
+        if (!SchemaConstants.ACCORD_KEYSPACE_NAME.equals(baseCfs.getKeyspaceName()))
+            throw new IllegalArgumentException("Route index is only allowed for accord commands table; given " + baseCfs().metadata());
+        if (!AccordKeyspace.COMMANDS.equals(baseCfs.name))
+            throw new IllegalArgumentException("Route index is only allowed for accord commands table; given " + baseCfs().metadata());
+
+        TableMetadata tableMetadata = baseCfs.metadata();
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, indexMetadata);
+        if (!AccordKeyspace.CommandsColumns.route.name.equals(target.left.name))
+            throw new IllegalArgumentException("Attempted to index the wrong column; needed " + AccordKeyspace.CommandsColumns.route.name + " but given " + target.left.name);
+
+        if (target.right != IndexTarget.Type.VALUES)
+            throw new IllegalArgumentException("Attempted to index " + AccordKeyspace.CommandsColumns.route.name + " with index type " + target.right + "; only " + IndexTarget.Type.VALUES + " is supported");
+
+        this.baseCfs = baseCfs;
+        this.indexMetadata = indexMetadata;
+        this.memtableIndexManager = new RouteMemtableIndexManager(this);
+        this.sstableManager = new RouteSSTableManager();
+        this.indexMetrics = new IndexMetrics(this);
+        this.column = target.left;
+
+        Tracker tracker = baseCfs.getTracker();
+        tracker.subscribe(this);
+    }
+
+    public ColumnMetadata column()
+    {
+        return column;
+    }
+
+    public IndexMetrics indexMetrics()
+    {
+        return indexMetrics;
+    }
+
+    public ColumnFamilyStore baseCfs()
+    {
+        return baseCfs;
+    }
+
+    @Override
+    public IndexMetadata getIndexMetadata()
+    {
+        return indexMetadata;
+    }
+
+    @Override
+    public boolean shouldBuildBlocking()
+    {
+        return true;
+    }
+
+    @Override
+    public boolean isSSTableAttached()
+    {
+        return true;
+    }
+
+    @Override
+    public Optional<ColumnFamilyStore> getBackingTable()
+    {
+        return Optional.empty();
+    }
+
+    @Override
+    public Set<Component> getComponents()
+    {
+        return Collections.singleton(type.getSingleton());
+    }
+
+    @Override
+    public Callable<?> getInitializationTask()
+    {
+        //TODO (now): in SAI startup doesn't validate... what are the downstream issues this can face?  Corrupt indexes not being detected?
+        boolean starting = StorageService.instance.isStarting();
+        return () -> {
+            if (baseCfs.indexManager.isIndexQueryable(this))
+            {
+                initBuildStarted = true;
+                return null;
+            }
+
+            // stop in-progress compaction tasks to prevent compacted sstable not being indexed.
+            CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()),
+                                                              ssTableReader -> true,
+                                                              true);
+            // Force another flush to make sure on disk index is generated for memtable data before marking it queryable.
+            // In the case of offline scrub, there are no live memtables.
+            if (!baseCfs.getTracker().getView().liveMemtables.isEmpty())
+                baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
+
+            // It is now safe to flush indexes directly from flushing Memtables.
+            initBuildStarted = true;
+
+            List<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, sstableManager);
+
+            if (nonIndexed.isEmpty())
+                return null;
+
+            // split sorted sstables into groups with similar size and build each group in separate compaction thread
+            List<List<SSTableReader>> groups = StorageAttachedIndex.groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentIndexBuilders());
+            List<Future<?>> futures = new ArrayList<>();
+
+            for (List<SSTableReader> group : groups)
+            {
+                futures.add(CompactionManager.instance.submitIndexBuild(new RouteSecondaryIndexBuilder(this, sstableManager, group, false, true)));
+            }
+
+            return FutureCombiner.allOf(futures).get();
+        };
+    }
+
+    private List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStore baseCfs, SSTableManager manager)
+    {
+        Set<SSTableReader> sstables = baseCfs.getLiveSSTables();
+
+        // Initialize the SSTable indexes w/ valid existing components...
+        manager.onSSTableChanged(Collections.emptyList(), sstables);
+
+        // ...then identify and rebuild the SSTable indexes that are missing.
+        List<SSTableReader> nonIndexed = new ArrayList<>();
+
+        for (SSTableReader sstable : sstables)
+        {
+            if (!sstable.isMarkedCompacted() && !manager.isIndexComplete(sstable))
+            {
+                nonIndexed.add(sstable);
+            }
+        }
+
+        return nonIndexed;
+    }
+
+
+    @Override
+    public boolean isQueryable(Status status)
+    {
+        // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes.
+        return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN;
+    }
+
+    public RegisterStatus registerStatus()
+    {
+        return registerStatus;
+    }
+
+    @Override
+    public synchronized void register(IndexRegistry registry)
+    {
+        registry.registerIndex(this);
+        registerStatus = RegisterStatus.REGISTERED;
+    }
+
+    @Override
+    public synchronized void unregister(IndexRegistry registry)
+    {
+        Index.super.unregister(registry);
+        registerStatus = RegisterStatus.UNREGISTERED;
+    }
+
+    @Override
+    public Callable<?> getTruncateTask(long truncatedAt)
+    {
+        /*
+         * index files will be removed as part of base sstable lifecycle in {@link LogTransaction#delete(java.io.File)}
+         * asynchronously, but we need to mark the index queryable because if the truncation is during the initial
+         * build of the index it won't get marked queryable by the build.
+         */
+        return () -> {
+            logger.info("Making index queryable during table truncation");
+            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+            return null;
+        };
+    }
+
+    @Override
+    public Callable<?> getBlockingFlushTask()
+    {
+        return null; // storage-attached indexes are flushed alongside memtable
+    }
+
+    @Override
+    public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
+    {
+        return null;
+    }
+
+    @Override
+    public Callable<?> getInvalidateTask()
+    {
+        return () -> null;
+    }
+
+    @Override
+    public void validate(PartitionUpdate update, ClientState state) throws InvalidRequestException
+    {
+        // only internal can write... so it must be valid no?
+    }
+
+
+    //TODO (now): flesh this stuff out...
+
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor,
+                                                 LifecycleNewTracker tracker)
+    {
+        // mimics org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat.newPerColumnIndexWriter
+        IndexDescriptor id = IndexDescriptor.create(descriptor, baseCfs.getPartitioner(), baseCfs.metadata().comparator);
+        if (tracker.opType() != OperationType.FLUSH || !initBuildStarted)
+        {
+            return new RouteIndexFormat.SSTableIndexWriter(this, id);
+        }
+        else
+        {
+            return new RouteIndexFormat.MemtableRouteIndexWriter(id, memtableIndexManager.getPendingMemtableIndex(tracker));
+        }
+    }
+
+    @Override
+    public Indexer indexerFor(DecoratedKey key,
+                              RegularAndStaticColumns columns,
+                              long nowInSec,
+                              WriteContext ctx,
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable)
+    {
+        // since we are attached we only care about update
+        if (transactionType != IndexTransaction.Type.UPDATE)
+            return null;
+        return new Indexer()
+        {
+            @Override
+            public void insertRow(Row row)
+            {
+                long size = memtableIndexManager.index(key, row, memtable);
+                if (size > 0)
+                    memtable.markExtraOnHeapUsed(size, CassandraWriteContext.fromContext(ctx).getGroup());
+            }
+
+            @Override
+            public void updateRow(Row oldRowData, Row newRowData)
+            {
+                insertRow(newRowData);
+            }
+        };
+    }
+
+    @Override
+    public boolean supportsExpression(ColumnMetadata column, Operator operator)
+    {
+        // disallow all queries, in order to interact with this index you must bypass CQL
+        return false;
+    }
+
+    @Override
+    public RowFilter getPostIndexQueryFilter(RowFilter filter)
+    {
+        return RowFilter.none();
+    }
+
+    @Override
+    public Searcher searcherFor(ReadCommand command)
+    {
+        List<RowFilter.Expression> expressions = command.rowFilter().getExpressions().stream().collect(Collectors.toList());
+        if (expressions.isEmpty())
+            return null;
+        ByteBuffer start = null;
+        boolean startInclusive = true;
+        ByteBuffer end = null;
+        boolean endInclusive = true;
+        Integer storeId = null;
+        for (RowFilter.Expression e : expressions)
+        {
+            if (e.column() == AccordKeyspace.CommandsColumns.route)
+            {
+                switch (e.operator())
+                {
+                    case GT:
+                        start = e.getIndexValue();
+                        startInclusive = false;
+                        break;
+                    case GTE:
+                        start = e.getIndexValue();
+                        startInclusive = true;
+                        break;
+                    case LT:
+                        end = e.getIndexValue();
+                        endInclusive = false;
+                        break;
+                    case LTE:
+                        end = e.getIndexValue();
+                        endInclusive = true;
+                        break;
+                    default:
+                        return null;
+                }
+            }
+            else if (e.column() == AccordKeyspace.CommandsColumns.store_id && e.operator() == Operator.EQ)
+            {
+                storeId = Int32Type.instance.compose(e.getIndexValue());
+            }
+        }
+        if (start == null || end == null || storeId == null)
+            return null;
+        int finalStoreId = storeId;
+        ByteBuffer finalStart = start;
+        boolean finalStartInclusive = startInclusive;
+        ByteBuffer finalEnd = end;
+        boolean finalEndInclusive = endInclusive;
+        return new Searcher()
+        {
+            @Override
+            public ReadCommand command()
+            {
+                return command;
+            }
+
+            @Override
+            public UnfilteredPartitionIterator search(ReadExecutionController executionController)
+            {
+                // find all partitions from memtable / sstable
+                NavigableSet<ByteBuffer> partitions = search(finalStoreId, finalStart, finalStartInclusive, finalEnd, finalEndInclusive);
+                // do SinglePartitionReadCommand per partition
+                return new SearchIterator(executionController, command, partitions);
+            }
+
+            NavigableSet<ByteBuffer> search(int storeId,
+                                              ByteBuffer startTableWithToken, boolean startInclusive,
+                                              ByteBuffer endTableWithToken, boolean endInclusive)
+            {
+                TableId tableId;
+                byte[] start;
+                {
+
+                    AccordRoutingKey route = OrderedRouteSerializer.deserializeRoutingKey(startTableWithToken);
+                    tableId = route.table();
+                    start = OrderedRouteSerializer.serializeRoutingKeyNoTable(route);
+                }
+                byte[] end = OrderedRouteSerializer.serializeRoutingKeyNoTable(OrderedRouteSerializer.deserializeRoutingKey(endTableWithToken));
+                NavigableSet<ByteBuffer> matches = sstableManager.search(storeId, tableId, start, startInclusive, end, endInclusive);
+                matches.addAll(memtableIndexManager.search(storeId, tableId, start, startInclusive, end, endInclusive));
+                return matches;
+            }
+        };
+    }
+
+    private class SearchIterator extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
+    {
+        private final ReadExecutionController executionController;
+        private final ReadCommand command;
+        private final TableMetadata metadata;
+        private final Iterator<ByteBuffer> partitions;
+
+        private SearchIterator(ReadExecutionController executionController, ReadCommand command, NavigableSet<ByteBuffer> partitions)
+        {
+            this.executionController = executionController;
+            this.command = command;
+            this.metadata = command.metadata();
+            this.partitions = partitions.iterator();
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return metadata;
+        }
+
+        @Override
+        protected UnfilteredRowIterator computeNext()
+        {
+            if (!partitions.hasNext())
+                return endOfData();
+            DecoratedKey pk = metadata.partitioner.decorateKey(partitions.next());
+            return new UnfilteredRowIterator()
+            {
+                @Override
+                public DeletionTime partitionLevelDeletion()
+                {
+                    return DeletionTime.LIVE;
+                }
+
+                @Override
+                public EncodingStats stats()
+                {
+                    return EncodingStats.NO_STATS;
+                }
+
+                @Override
+                public TableMetadata metadata()
+                {
+                    return metadata;
+                }
+
+                @Override
+                public boolean isReverseOrder()
+                {
+                    return false;
+                }
+
+                @Override
+                public RegularAndStaticColumns columns()
+                {
+                    return RegularAndStaticColumns.NONE;
+                }
+
+                @Override
+                public DecoratedKey partitionKey()
+                {
+                    return pk;
+                }
+
+                @Override
+                public Row staticRow()
+                {
+                    return null;
+                }
+
+                @Override
+                public void close()
+                {
+
+                }
+
+                private Row row = BTreeRow.emptyRow(Clustering.EMPTY);
+
+                @Override
+                public boolean hasNext()
+                {
+                    return row != null;
+                }
+
+                @Override
+                public Unfiltered next()
+                {
+                    Row row = this.row;
+                    this.row = null;
+                    return row;
+                }
+            };
+        }
+
+        @Override
+        public void close()
+        {
+
+        }
+    }
+
+    @Override
+    public void handleNotification(INotification notification, Object sender)
+    {
+        // unfortunately, we can only check the type of notification via instanceof :(
+        if (notification instanceof SSTableAddedNotification)
+        {
+            SSTableAddedNotification notice = (SSTableAddedNotification) notification;
+            sstableManager.onSSTableChanged(Collections.emptySet(), notice.added);
+        }
+        else if (notification instanceof SSTableListChangedNotification)
+        {
+            SSTableListChangedNotification notice = (SSTableListChangedNotification) notification;
+            sstableManager.onSSTableChanged(notice.removed, notice.added);
+        }
+        else if (notification instanceof MemtableRenewedNotification)
+        {
+            memtableIndexManager.renewMemtable(((MemtableRenewedNotification) notification).renewed);
+        }
+        else if (notification instanceof MemtableDiscardedNotification)
+        {
+            memtableIndexManager.discardMemtable(((MemtableDiscardedNotification) notification).memtable);
+        }
+    }
+
+    //TODO (coverage): everything below here never triggered...
+
+    @Override
+    public boolean dependsOn(ColumnMetadata column)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public AbstractType<?> customExpressionValueType()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        throw new UnsupportedOperationException();
+    }
+}

src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java

@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+import com.google.common.collect.Maps;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.index.accord.IndexDescriptor.IndexComponent;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.util.ChecksumedRandomAccessReader;
+import org.apache.cassandra.io.util.ChecksumedSequentialWriter;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.serializers.UUIDSerializer;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.utils.Clock.Global.nowInSeconds;
+
+// A route index consists of a few files: cintia_sorted_list, cintia_checkpoints, and metadata
+// metadata stores the segement mappings and stats needed for search selection
+public class RouteIndexFormat
+{
+    public static final Supplier<Checksum> CHECKSUM_SUPPLIER = CRC32C::new;
+
+    public interface Writer extends SSTableFlushObserver
+    {
+
+    }
+
+    public static class SSTableIndexWriter extends MemtableRouteIndexWriter
+    {
+        private final RouteIndex index;
+        private DecoratedKey current;
+
+        public SSTableIndexWriter(RouteIndex index, IndexDescriptor id)
+        {
+            super(id, new MemtableIndex());
+            this.index = index;
+        }
+
+        @Override
+        public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
+        {
+            this.current = key;
+        }
+
+        @Override
+        public void nextUnfilteredCluster(Unfiltered unfiltered)
+        {
+            // there is some duplication from org.apache.cassandra.index.accord.RouteMemtableIndexManager.index
+            // should this be cleaned up?
+            if (!unfiltered.isRow())
+                return;
+            Row row = (Row) unfiltered;
+            // simplified version of org.apache.cassandra.index.sai.utils.IndexTermType.valueOf
+            Cell<?> cell = row.getCell(index.column());
+            ByteBuffer value = cell == null || !cell.isLive(nowInSeconds()) ? null : cell.buffer();
+            indexer.index(current, row.clustering(), value);
+        }
+    }
+
+    public static class MemtableRouteIndexWriter implements Writer
+    {
+        private final IndexDescriptor id;
+        protected final MemtableIndex indexer;
+
+        public MemtableRouteIndexWriter(IndexDescriptor id, MemtableIndex indexer)
+        {
+            this.id = id;
+            this.indexer = indexer;
+        }
+
+
+        @Override
+        public void begin()
+        {
+            // no-op
+        }
+
+        @Override
+        public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
+        {
+            // no-op
+        }
+
+        @Override
+        public void staticRow(Row staticRow)
+        {
+            // no-op
+        }
+
+        @Override
+        public void nextUnfilteredCluster(Unfiltered unfiltered)
+        {
+            // no-op
+        }
+
+        @Override
+        public void complete()
+        {
+            try
+            {
+                if (!indexer.isEmpty())
+                {
+                    Segment segment = indexer.write(id);
+                    appendSegment(id, segment);
+                }
+                else
+                {
+                    // nothing to see here... need to still mark the SSTable as indexed, so need an empty segment
+                    appendSegment(id, Segment.EMPTY);
+                }
+            }
+            catch (IOException e)
+            {
+                abort(e);
+                throw Throwables.unchecked(e);
+            }
+        }
+
+        @Override
+        public void abort(Throwable accumulator)
+        {
+            id.deleteIndex();
+        }
+
+        public void abort(Throwable accumulator, boolean fromIndex)
+        {
+            abort(accumulator);
+            // If the abort was from an index error, propagate the error upstream so index builds, compactions, and
+            // flushes can handle it correctly.
+            if (fromIndex)
+                throw Throwables.unchecked(accumulator);
+        }
+    }
+
+    static List<Segment> readSegements(Map<IndexComponent, FileHandle> index) throws IOException
+    {
+        List<Segment> segments = new ArrayList<>();
+
+        try (var metaReader = new ChecksumedRandomAccessReader(index.get(IndexComponent.METADATA).createReader(), CHECKSUM_SUPPLIER);
+             var segmentReader = new ChecksumedRandomAccessReader(index.get(IndexComponent.SEGMENT).createReader(), CHECKSUM_SUPPLIER))
+        {
+            while (metaReader.getFilePointer() < metaReader.length())
+            {
+                metaReader.resetChecksum();
+                long startPointer = metaReader.readUnsignedVInt();
+                long endPointer = metaReader.readUnsignedVInt();
+                int groupSize = metaReader.readUnsignedVInt32();
+                int segmentChecksum = metaReader.readInt();
+                int metadataChecksum = metaReader.getValue32AndResetChecksum();
+                int actualChecksum = metaReader.readInt();
+                assert actualChecksum == metadataChecksum;
+
+                segmentReader.resetChecksum();
+                segmentReader.seek(startPointer);
+                Map<Group, Segment.Metadata> groups = Maps.newHashMapWithExpectedSize(groupSize);
+                for (int i = 0; i < groupSize; i++)
+                {
+                    int storeId = segmentReader.readVInt32();
+                    TableId tableId = TableId.fromUUID(new UUID(segmentReader.readLong(), segmentReader.readLong()));
+                    Group group = new Group(storeId, tableId);
+                    int metaSize = segmentReader.readUnsignedVInt32();
+                    EnumMap<IndexComponent, Segment.ComponentMetadata> metas = new EnumMap<>(IndexComponent.class);
+                    for (int j = 0; j < metaSize; j++)
+                    {
+                        IndexComponent c = IndexComponent.fromByte(segmentReader.readByte());
+                        metas.put(c, new Segment.ComponentMetadata(segmentReader.readUnsignedVInt(), segmentReader.readUnsignedVInt()));
+                    }
+                    byte[] minTerm = ByteArrayUtil.readWithVIntLength(segmentReader);
+                    byte[] maxTerm = ByteArrayUtil.readWithVIntLength(segmentReader);
+                    Segment.Metadata existing = groups.put(group, new Segment.Metadata(metas, minTerm, maxTerm));
+                    assert existing == null;
+                }
+                int actualSegmentChecksum = segmentReader.getValue32AndResetChecksum();
+                assert actualSegmentChecksum == segmentChecksum;
+                assert segmentReader.getFilePointer() == endPointer;
+                segments.add(new Segment(groups));
+            }
+        }
+        return segments;
+    }
+
+    static void appendSegment(IndexDescriptor id, Segment segment) throws IOException
+    {
+        List<Group> groups = new ArrayList<>(segment.groups.keySet());
+        groups.sort(Comparator.naturalOrder());
+
+        try (var segmentWriter = ChecksumedSequentialWriter.open(id.fileFor(IndexComponent.SEGMENT), true, CHECKSUM_SUPPLIER);
+             var metadataWriter = ChecksumedSequentialWriter.open(id.fileFor(IndexComponent.METADATA), true, CHECKSUM_SUPPLIER))
+        {
+            long startPointer = segmentWriter.getFilePointer();
+            for (Group group : groups)
+            {
+                Segment.Metadata metadata = segment.groups.get(group);
+                writeGroup(segmentWriter, group, metadata);
+            }
+            long endPointer = segmentWriter.getFilePointer();
+
+            int checksum = segmentWriter.getValue32AndResetChecksum();
+            metadataWriter.writeUnsignedVInt(startPointer);
+            metadataWriter.writeUnsignedVInt(endPointer);
+            metadataWriter.writeUnsignedVInt32(segment.groups.size());
+            metadataWriter.writeInt(checksum);
+            metadataWriter.writeInt(metadataWriter.getValue32AndResetChecksum());
+        }
+    }
+
+    private static void writeGroup(ChecksumedSequentialWriter seq, Group group, Segment.Metadata metadata) throws IOException
+    {
+        seq.writeVInt32(group.storeId);
+        seq.write(UUIDSerializer.instance.serialize(group.tableId.asUUID()));
+        seq.writeUnsignedVInt32(metadata.metas.size());
+        for (Map.Entry<IndexComponent, Segment.ComponentMetadata> e : metadata.metas.entrySet())
+        {
+            seq.writeByte(e.getKey().value);
+            seq.writeUnsignedVInt(e.getValue().offset);
+            seq.writeUnsignedVInt(e.getValue().endOffset);
+        }
+        ByteArrayUtil.writeWithVIntLength(metadata.minTerm, seq);
+        ByteArrayUtil.writeWithVIntLength(metadata.maxTerm, seq);
+    }
+}

src/java/org/apache/cassandra/index/accord/RouteMemtableIndexManager.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.nio.ByteBuffer;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.schema.TableId;
+
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+import static org.apache.cassandra.utils.Clock.Global.nowInSeconds;
+
+public class RouteMemtableIndexManager implements MemtableIndexManager
+{
+    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap = new ConcurrentHashMap<>();
+    private final RouteIndex index;
+
+    public RouteMemtableIndexManager(RouteIndex index)
+    {
+        this.index = index;
+    }
+
+    @Override
+    public long index(DecoratedKey key, Row row, Memtable mt)
+    {
+        if (row.isStatic())
+            return 0;
+        //TODO (performance): we dropped jdk8 and this was fixed in jdk8... so do we need to do this still?
+        MemtableIndex current = liveMemtableIndexMap.get(mt);
+
+        // We expect the relevant IndexMemtable to be present most of the time, so only make the
+        // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
+        MemtableIndex target = (current != null)
+                               ? current
+                               : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex());
+
+        long start = nanoTime();
+
+        long bytes = 0;
+
+        // simplified version of org.apache.cassandra.index.sai.utils.IndexTermType.valueOf
+        Cell<?> cell = row.getCell(index.column());
+        ByteBuffer value = cell == null || !cell.isLive(nowInSeconds()) ? null : cell.buffer();
+
+        bytes += target.index(key, row.clustering(), value);
+        index.indexMetrics().memtableIndexWriteLatency.update(nanoTime() - start, TimeUnit.NANOSECONDS);
+        return bytes;
+    }
+
+    @Override
+    public MemtableIndex getPendingMemtableIndex(LifecycleNewTracker tracker)
+    {
+        return liveMemtableIndexMap.keySet().stream()
+                                   .filter(m -> tracker.equals(m.getFlushTransaction()))
+                                   .findFirst()
+                                   .map(liveMemtableIndexMap::get)
+                                   .orElse(null);
+    }
+
+    @Override
+    public void discardMemtable(Memtable memtable)
+    {
+        liveMemtableIndexMap.remove(memtable);
+    }
+
+    @Override
+    public void renewMemtable(Memtable renewed)
+    {
+        for (Memtable memtable : liveMemtableIndexMap.keySet())
+        {
+            // remove every index but the one that corresponds to the post-truncate Memtable
+            if (renewed != memtable)
+            {
+                liveMemtableIndexMap.remove(memtable);
+            }
+        }
+    }
+
+    @Override
+    public NavigableSet<ByteBuffer> search(int storeId, TableId tableId, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive)
+    {
+        TreeSet<ByteBuffer> matches = new TreeSet<>();
+        liveMemtableIndexMap.values().forEach(m -> matches.addAll(m.search(storeId, tableId, start, startInclusive, end, endInclusive)));
+        return matches;
+    }
+}

src/java/org/apache/cassandra/index/accord/RouteSSTableManager.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
+
+public class RouteSSTableManager implements SSTableManager
+{
+    private final Map<SSTableReader, SSTableIndex> sstables = new HashMap<>();
+
+    @Override
+    public synchronized void onSSTableChanged(Collection<SSTableReader> removed, Iterable<SSTableReader> added)
+    {
+        //TODO (performance): most added tables will have 0 segmenets, so exclude those from search
+        removed.forEach(s -> {
+            SSTableIndex index = sstables.remove(s);
+            if (index != null)
+            {
+                index.close();
+                index.id.deleteIndex();
+            }
+        });
+
+        List<SSTableReader> notComplete = null;
+        for (SSTableReader sstable : added)
+        {
+            IndexDescriptor id = IndexDescriptor.create(sstable);
+            if (!id.isIndexBuildComplete())
+            {
+                if (notComplete == null) notComplete = new ArrayList<>();
+                notComplete.add(sstable);
+                continue;
+            }
+            try
+            {
+                sstables.put(sstable, SSTableIndex.create(id));
+            }
+            catch (IOException e)
+            {
+                notComplete.add(sstable);
+            }
+        }
+        if (notComplete != null)
+            throw new IllegalArgumentException("SStables were added without an index... " + notComplete);
+    }
+
+    @Override
+    public synchronized boolean isIndexComplete(SSTableReader reader)
+    {
+        return sstables.containsKey(reader);
+    }
+
+    @Override
+    public synchronized NavigableSet<ByteBuffer> search(int storeId, TableId tableId,
+                                                        byte[] start, boolean startInclusive,
+                                                        byte[] end, boolean endInclusive)
+    {
+        Group group = new Group(storeId, tableId);
+        TreeSet<ByteBuffer> matches = new TreeSet<>();
+        for (SSTableIndex index : sstables.values())
+            matches.addAll(index.search(group, start, startInclusive, end, endInclusive));
+        return matches;
+    }
+}

src/java/org/apache/cassandra/index/accord/RouteSecondaryIndexBuilder.java

@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
+{
+    private static final Logger logger = LoggerFactory.getLogger(RouteSecondaryIndexBuilder.class);
+
+    private final TimeUUID compactionId = nextTimeUUID();
+    private final RouteIndex index;
+    private final TableMetadata metadata;
+    private final Tracker tracker;
+    private final SSTableManager sstableManager;
+    private final List<SSTableReader> sstables;
+    private final boolean isFullRebuild;
+    private final boolean isInitialBuild;
+    private final long totalSizeInBytes;
+    private long bytesProcessed = 0;
+
+    public RouteSecondaryIndexBuilder(RouteIndex index,
+                                      SSTableManager sstableManager,
+                                      List<SSTableReader> sstables,
+                                      boolean isFullRebuild,
+                                      boolean isInitialBuild)
+    {
+        this.index = index;
+        this.metadata = index.baseCfs().metadata();
+        this.tracker = index.baseCfs().getTracker();
+        this.sstableManager = sstableManager;
+        this.sstables = sstables;
+        this.isFullRebuild = isFullRebuild;
+        this.isInitialBuild = isInitialBuild;
+        this.totalSizeInBytes = sstables.stream().mapToLong(SSTableReader::uncompressedLength).sum();
+    }
+
+    @Override
+    public CompactionInfo getCompactionInfo()
+    {
+        return new CompactionInfo(metadata,
+                                  OperationType.INDEX_BUILD,
+                                  bytesProcessed,
+                                  totalSizeInBytes,
+                                  compactionId,
+                                  sstables);
+    }
+
+    @Override
+    public void build()
+    {
+        if (!validateIndexes())
+            return;
+        for (SSTableReader sstable : sstables)
+        {
+            if (indexSSTable(sstable))
+                return;
+        }
+    }
+
+    /**
+     * @return true if index build should be stopped
+     */
+    private boolean indexSSTable(SSTableReader sstable)
+    {
+        logger.debug("Starting index build on {}", sstable.descriptor);
+
+        RouteIndexFormat.SSTableIndexWriter indexWriter = null;
+
+        Ref<? extends SSTableReader> ref = sstable.tryRef();
+        if (ref == null)
+        {
+            logger.warn("Couldn't acquire reference to the SSTable {}. It may have been removed.", sstable.descriptor);
+            return false;
+        }
+
+        try (RandomAccessReader dataFile = sstable.openDataReader();
+             LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.INDEX_BUILD, sstable))
+        {
+            // remove existing per column index files instead of overwriting
+            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+            indexDescriptor.deleteIndex();
+
+            indexWriter = new RouteIndexFormat.SSTableIndexWriter(index, indexDescriptor);
+            indexWriter.begin();
+
+            long previousBytesRead = 0;
+
+            try (KeyIterator keys = sstable.keyIterator())
+            {
+                while (keys.hasNext())
+                {
+                    if (isStopRequested())
+                    {
+                        logger.debug("Index build has been stopped");
+                        throw new CompactionInterruptedException(getCompactionInfo());
+                    }
+
+                    DecoratedKey key = keys.next();
+
+                    indexWriter.startPartition(key, -1, -1);
+
+                    long position = sstable.getPosition(key, SSTableReader.Operator.EQ);
+                    dataFile.seek(position);
+                    ByteBufferUtil.readWithShortLength(dataFile); // key
+
+                    try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key))
+                    {
+                        // if the row has statics attached, it has to be indexed separately
+                        if (metadata.hasStaticColumns())
+                            indexWriter.nextUnfilteredCluster(partition.staticRow());
+
+                        while (partition.hasNext())
+                            indexWriter.nextUnfilteredCluster(partition.next());
+                    }
+                    long bytesRead = keys.getBytesRead();
+                    bytesProcessed += bytesRead - previousBytesRead;
+                    previousBytesRead = bytesRead;
+                }
+
+                completeSSTable(indexDescriptor, indexWriter, sstable);
+            }
+
+            return false;
+        }
+        catch (Throwable t)
+        {
+            if (indexWriter != null)
+            {
+                indexWriter.abort(t, true);
+            }
+
+            if (t instanceof InterruptedException)
+            {
+                logger.warn("Interrupted while building indexes on SSTable {}", sstable.descriptor);
+                Thread.currentThread().interrupt();
+                return true;
+            }
+            else if (t instanceof CompactionInterruptedException)
+            {
+                //TODO Shouldn't do this if the stop was interrupted by a truncate
+                if (isInitialBuild)
+                {
+                    logger.error("Stop requested while building initial indexes on SSTable {}.", sstable.descriptor);
+                    throw Throwables.unchecked(t);
+                }
+                else
+                {
+                    logger.info("Stop requested while building indexes on SSTable {}.", sstable.descriptor);
+                    return true;
+                }
+            }
+            else
+            {
+                logger.error("Unable to build indexes on SSTable {}. Cause: {}.", sstable, t.getMessage());
+                throw Throwables.unchecked(t);
+            }
+        }
+        finally
+        {
+            ref.release();
+        }
+    }
+
+    private void completeSSTable(IndexDescriptor indexDescriptor, SSTableFlushObserver indexWriter, SSTableReader sstable)
+    {
+        indexWriter.complete();
+
+        if (!validateIndexes())
+        {
+            logger.debug("dropped during index build");
+            return;
+        }
+
+        // register custom index components into existing sstables
+        sstable.registerComponents(indexDescriptor.getLiveSSTableComponents(), tracker);
+        sstableManager.onSSTableChanged(Collections.emptyList(), Collections.singleton(sstable));
+    }
+
+    /**
+     * In case of full rebuild, stop the index build if any index is dropped.
+     * Otherwise, skip dropped indexes to avoid exception during repair/streaming.
+     */
+    private boolean validateIndexes()
+    {
+        switch (index.registerStatus())
+        {
+            case PENDING: throw new IllegalStateException("Unable to build indexes if the index has not been registered");
+            case REGISTERED: return true;
+            case UNREGISTERED: break;
+            default: throw new AssertionError("Unknown status: " + index.registerStatus());
+        }
+
+        if (isFullRebuild)
+            throw new RuntimeException(String.format("%s are dropped, will stop index build.", index.getIndexMetadata().name));
+
+        return false;
+    }
+}

src/java/org/apache/cassandra/index/accord/RoutesSearcher.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import accord.primitives.TxnId;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class RoutesSearcher
+{
+    private final ColumnFamilyStore cfs = Keyspace.open("system_accord").getColumnFamilyStore("commands");
+    private final Index index = cfs.indexManager.getIndexByName("route");;
+    private final ColumnMetadata route = AccordKeyspace.CommandsColumns.route;
+    private final ColumnMetadata store_id = AccordKeyspace.CommandsColumns.store_id;
+    private final ColumnMetadata txn_id = AccordKeyspace.CommandsColumns.txn_id;
+    private final ColumnFilter columnFilter = ColumnFilter.selectionBuilder().add(store_id).add(txn_id).build();
+    private final DataLimits limits = DataLimits.NONE;
+    private final DataRange dataRange = DataRange.allData(cfs.getPartitioner());
+
+    private CloseableIterator<Entry> searchKeysAccord(int store, AccordRoutingKey start, AccordRoutingKey end)
+    {
+        RowFilter rowFilter = RowFilter.create(false);
+        rowFilter.add(route, Operator.GT, OrderedRouteSerializer.serializeRoutingKey(start));
+        rowFilter.add(route, Operator.LTE, OrderedRouteSerializer.serializeRoutingKey(end));
+        rowFilter.add(store_id, Operator.EQ, Int32Type.instance.decompose(store));
+
+        PartitionRangeReadCommand cmd = PartitionRangeReadCommand.create(cfs.metadata(),
+                                                                         FBUtilities.nowInSeconds(),
+                                                                         columnFilter,
+                                                                         rowFilter,
+                                                                         limits,
+                                                                         dataRange);
+        Index.Searcher s = index.searcherFor(cmd);
+        try (ReadExecutionController controler = cmd.executionController())
+        {
+            UnfilteredPartitionIterator partitionIterator = s.search(controler);
+            return new CloseableIterator<Entry>()
+            {
+                private final Entry entry = new Entry();
+                @Override
+                public void close()
+                {
+                    partitionIterator.close();
+                }
+
+                @Override
+                public boolean hasNext()
+                {
+                    return partitionIterator.hasNext();
+                }
+
+                @Override
+                public Entry next()
+                {
+                    UnfilteredRowIterator next = partitionIterator.next();
+                    ByteBuffer[] partitionKeyComponents = AccordKeyspace.CommandRows.splitPartitionKey(next.partitionKey());
+                    entry.store_id = AccordKeyspace.CommandRows.getStoreId(partitionKeyComponents);
+                    entry.txnId = AccordKeyspace.CommandRows.getTxnId(partitionKeyComponents);
+                    return entry;
+                }
+            };
+        }
+    }
+
+    public Set<TxnId> intersects(int store, TokenRange range)
+    {
+        return intersects(store, (AccordRoutingKey) range.start(), (AccordRoutingKey) range.end());
+    }
+
+    public Set<TxnId> intersects(int store, AccordRoutingKey start, AccordRoutingKey end)
+    {
+        HashSet<TxnId> set = new HashSet<TxnId>();
+        try (CloseableIterator<Entry> it = searchKeysAccord(store, start, end))
+        {
+            while (it.hasNext())
+            {
+                Entry next = it.next();
+                if (next.store_id != store) continue; // the index should filter out, but just in case...
+                set.add(next.txnId);
+            }
+        }
+        return set.isEmpty() ? Collections.emptySet() : set;
+    }
+
+    private static final class Entry
+    {
+        public int store_id;
+        public TxnId txnId;
+    }
+}

src/java/org/apache/cassandra/index/accord/SSTableIndex.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.index.accord.CheckpointIntervalArrayIndex.SegmentSearcher;
+import org.apache.cassandra.index.accord.IndexDescriptor.IndexComponent;
+import org.apache.cassandra.io.FSReadError;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
+
+public class SSTableIndex extends SharedCloseableImpl
+{
+    public final IndexDescriptor id;
+    private final Map<IndexComponent, FileHandle> files;
+    private final List<Segment> segments;
+
+    private SSTableIndex(IndexDescriptor id,
+                         Map<IndexComponent, FileHandle> files,
+                         List<Segment> segments,
+                         Cleanup cleanup)
+    {
+        super(cleanup);
+        this.id = id;
+        this.files = files;
+        this.segments = segments;
+    }
+
+    public SSTableIndex(SSTableIndex copy)
+    {
+        super(copy);
+        this.id = copy.id;
+        this.files = copy.files;
+        this.segments = copy.segments;
+    }
+
+    @Override
+    public SSTableIndex sharedCopy()
+    {
+        return new SSTableIndex(this);
+    }
+
+    public static SSTableIndex create(IndexDescriptor id) throws IOException
+    {
+        Map<IndexComponent, FileHandle> files = new EnumMap<>(IndexComponent.class);
+        for (IndexComponent c : id.getLiveComponents())
+            files.put(c, new FileHandle.Builder(id.fileFor(c)).mmapped(true).complete());
+        List<Segment> segments = RouteIndexFormat.readSegements(files);
+        files.remove(IndexComponent.SEGMENT).close();
+        files.remove(IndexComponent.METADATA).close();
+        Cleanup cleanup = new Cleanup(files);
+        return new SSTableIndex(id, files, segments, cleanup);
+    }
+
+    public Collection<? extends ByteBuffer> search(Group group, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive)
+    {
+        List<Segment> matches = segments.stream().filter(s -> {
+                                            Segment.Metadata metadata = s.groups.get(group);
+                                            if (metadata == null) return false;
+                                            if (ByteArrayUtil.compareUnsigned(metadata.minTerm, end) >= 0)
+                                                return false;
+                                            if (ByteArrayUtil.compareUnsigned(metadata.maxTerm, start) <= 0)
+                                                return false;
+                                            return true;
+                                        })
+                                        .collect(Collectors.toList());
+        if (matches.isEmpty()) return Collections.emptyList();
+        if (matches.size() == 1) return search(matches.get(0), group, start, startInclusive, end, endInclusive);
+        Set<ByteBuffer> found =  new HashSet<>();
+        for (Segment s : matches)
+            found.addAll(search(s, group, start, startInclusive, end, endInclusive));
+        return found;
+    }
+
+    private Collection<? extends ByteBuffer> search(Segment segment, Group group, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive)
+    {
+        Set<ByteBuffer> matches = new HashSet<>();
+        Segment.Metadata metadata = Objects.requireNonNull(segment.groups.get(group), () -> "Unknown group: " + group);
+        try
+        {
+            SegmentSearcher searcher = new SegmentSearcher(fileFor(IndexComponent.CINTIA_SORTED_LIST), metadata.metas.get(IndexComponent.CINTIA_SORTED_LIST).offset,
+                                                           fileFor(IndexComponent.CINTIA_CHECKPOINTS), metadata.metas.get(IndexComponent.CINTIA_CHECKPOINTS).offset);
+            searcher.intersects(start, end, interval -> matches.add(ByteBuffer.wrap(interval.value)));
+        }
+        catch (IOException e)
+        {
+            throw new FSReadError(e, id.fileFor(IndexComponent.CINTIA_SORTED_LIST));
+        }
+        return matches;
+    }
+
+    private FileHandle fileFor(IndexComponent c)
+    {
+        return Objects.requireNonNull(files.get(c), () -> "Unknown component: " + c);
+    }
+
+    private static class Cleanup implements RefCounted.Tidy
+    {
+        private final Map<IndexComponent, FileHandle> files;
+
+        private Cleanup(Map<IndexComponent, FileHandle> files)
+        {
+            this.files = files;
+        }
+
+        @Override
+        public void tidy() throws Exception
+        {
+            for (IndexComponent c : IndexComponent.values())
+            {
+                FileHandle fh = files.remove(c);
+                if (fh == null) continue;
+                fh.close();
+            }
+        }
+
+        @Override
+        public String name()
+        {
+            return "SSTableIndex Cleanup";
+        }
+    }
+}

src/java/org/apache/cassandra/index/accord/SSTableManager.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.NavigableSet;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableId;
+
+public interface SSTableManager
+{
+    void onSSTableChanged(Collection<SSTableReader> removed, Iterable<SSTableReader> added);
+    boolean isIndexComplete(SSTableReader reader);
+
+    NavigableSet<ByteBuffer> search(int storeId, TableId tableId, byte[] start, boolean startInclusive, byte[] end, boolean endInclusive);
+}

src/java/org/apache/cassandra/index/accord/Segment.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
+
+public class Segment
+{
+    public static final Segment EMPTY = new Segment(Collections.emptyMap());
+
+    public final Map<Group, Metadata> groups;
+
+    public Segment(Map<Group, Metadata> groups)
+    {
+        this.groups = groups;
+    }
+
+    public static class ComponentMetadata
+    {
+        public final long offset, endOffset;
+
+        public ComponentMetadata(long offset, long endOffset)
+        {
+            this.offset = offset;
+            this.endOffset = endOffset;
+        }
+    }
+
+    public static class Metadata
+    {
+        public final EnumMap<IndexDescriptor.IndexComponent, ComponentMetadata> metas;
+        public final byte[] minTerm, maxTerm;
+
+        public Metadata(EnumMap<IndexDescriptor.IndexComponent, ComponentMetadata> metas, byte[] minTerm, byte[] maxTerm)
+        {
+            this.metas = metas;
+            this.minTerm = minTerm;
+            this.maxTerm = maxTerm;
+        }
+    }
+}

src/java/org/apache/cassandra/io/util/Checksumed.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.util.zip.Checksum;
+
+public interface Checksumed
+{
+    Checksum checksum();
+
+    default long getAndResetChecksum()
+    {
+        Checksum c = checksum();
+        long v = c.getValue();
+        c.reset();
+        return v;
+    }
+
+    default int getValue32()
+    {
+        long v = checksum().getValue();
+        if (Long.numberOfLeadingZeros(v) < 32)
+            throw new IllegalStateException("Checksum is larger than 32 bytes!");
+        return (int) v;
+    }
+
+    default int getValue32AndResetChecksum()
+    {
+        Checksum c = checksum();
+        long v = c.getValue();
+        if (Long.numberOfLeadingZeros(v) < 32)
+            throw new IllegalStateException("Checksum is larger than 32 bytes!");
+        c.reset();
+        return (int) v;
+    }
+
+    default void resetChecksum()
+    {
+        checksum().reset();
+    }
+}

src/java/org/apache/cassandra/io/util/ChecksumedDataInputPlus.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+import java.util.zip.Checksum;
+
+public class ChecksumedDataInputPlus implements DataInputPlus, Checksumed
+{
+    private final DataInputPlus delegate;
+    private final Checksum checksum;
+
+    public ChecksumedDataInputPlus(DataInputPlus delegate, Checksum checksum)
+    {
+        this.delegate = delegate;
+        this.checksum = checksum;
+    }
+
+    public ChecksumedDataInputPlus(DataInputPlus delegate, Supplier<Checksum> fn)
+    {
+        this(delegate, fn.get());
+    }
+
+    public DataInputPlus delegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public Checksum checksum()
+    {
+        return checksum;
+    }
+
+    @Override
+    public void readFully(byte[] b) throws IOException
+    {
+        delegate().readFully(b);
+        checksum.update(b);
+    }
+
+    @Override
+    public void readFully(byte[] b, int off, int len) throws IOException
+    {
+        delegate().readFully(b, off, len);
+        checksum.update(b, off, len);
+    }
+
+    @Override
+    public int skipBytes(int n) throws IOException
+    {
+        int skipped = delegate().skipBytes(n);
+        checksum.reset();
+        return skipped;
+    }
+
+    @Override
+    public int readUnsignedByte() throws IOException
+    {
+        int value = delegate().readUnsignedByte();
+        checksum.update(value);
+        return value;
+    }
+
+    private byte writeBuffer[] = new byte[8];
+
+    @Override
+    public long readLong() throws IOException
+    {
+        long v = delegate().readLong();
+        writeBuffer[0] = (byte)(v >>> 56);
+        writeBuffer[1] = (byte)(v >>> 48);
+        writeBuffer[2] = (byte)(v >>> 40);
+        writeBuffer[3] = (byte)(v >>> 32);
+        writeBuffer[4] = (byte)(v >>> 24);
+        writeBuffer[5] = (byte)(v >>> 16);
+        writeBuffer[6] = (byte)(v >>>  8);
+        writeBuffer[7] = (byte)(v >>>  0);
+        checksum.update(writeBuffer);
+        return v;
+    }
+
+    @Override
+    public String readLine() throws IOException
+    {
+        throw new UnsupportedOperationException();
+    }
+}

src/java/org/apache/cassandra/io/util/ChecksumedDataOutputPlus.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.function.Supplier;
+import java.util.zip.Checksum;
+
+public class ChecksumedDataOutputPlus implements DataOutputPlus, Checksumed
+{
+    private final DataOutputPlus delegate;
+    private final Checksum checksum;
+
+    public ChecksumedDataOutputPlus(DataOutputPlus delegate, Checksum checksum)
+    {
+        this.delegate = delegate;
+        this.checksum = checksum;
+    }
+
+    public ChecksumedDataOutputPlus(DataOutputPlus delegate, Supplier<Checksum> fn)
+    {
+        this(delegate, fn.get());
+    }
+
+    public DataOutputPlus delegate()
+    {
+        return delegate;
+    }
+
+    @Override
+    public Checksum checksum()
+    {
+        return checksum;
+    }
+
+    @Override
+    public void write(ByteBuffer buffer) throws IOException
+    {
+        checksum.update(buffer.duplicate());
+        delegate().write(buffer);
+    }
+
+    @Override
+    public void write(int b) throws IOException
+    {
+        checksum.update(b);
+        delegate().write(b);
+    }
+
+    @Override
+    public void write(byte[] b) throws IOException
+    {
+        checksum.update(b);
+        delegate().write(b);
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException
+    {
+        checksum.update(b, off, len);
+        delegate().write(b, off, len);
+    }
+
+    @Override
+    public void writeByte(int v) throws IOException
+    {
+        checksum.update(v);
+        delegate().writeByte(v);
+    }
+
+    private byte writeBuffer[] = new byte[8];
+
+    @Override
+    public void writeLong(long v) throws IOException
+    {
+        writeBuffer[0] = (byte)(v >>> 56);
+        writeBuffer[1] = (byte)(v >>> 48);
+        writeBuffer[2] = (byte)(v >>> 40);
+        writeBuffer[3] = (byte)(v >>> 32);
+        writeBuffer[4] = (byte)(v >>> 24);
+        writeBuffer[5] = (byte)(v >>> 16);
+        writeBuffer[6] = (byte)(v >>>  8);
+        writeBuffer[7] = (byte)(v >>>  0);
+        checksum.update(writeBuffer);
+        delegate().writeLong(v);
+    }
+
+    @Override
+    public void writeUTF(String s) throws IOException
+    {
+        throw new UnsupportedOperationException("TODO");
+    }
+}

src/java/org/apache/cassandra/io/util/ChecksumedFileDataInput.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+import java.util.zip.Checksum;
+
+public class ChecksumedFileDataInput extends ChecksumedDataInputPlus implements FileDataInput
+{
+    public ChecksumedFileDataInput(FileDataInput delegate, Checksum checksum)
+    {
+        super(delegate, checksum);
+    }
+
+    public ChecksumedFileDataInput(FileDataInput delegate, Supplier<Checksum> fn)
+    {
+        super(delegate, fn);
+    }
+
+    @Override
+    public FileDataInput delegate()
+    {
+        return (FileDataInput) super.delegate();
+    }
+
+    @Override
+    public String getPath()
+    {
+        return delegate().getPath();
+    }
+
+    @Override
+    public boolean isEOF() throws IOException
+    {
+        return delegate().isEOF();
+    }
+
+    @Override
+    public long bytesRemaining() throws IOException
+    {
+        return delegate().bytesRemaining();
+    }
+
+    @Override
+    public void seek(long pos) throws IOException
+    {
+        resetChecksum();
+        delegate().seek(pos);
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return delegate().getFilePointer();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        resetChecksum();
+        delegate().close();
+    }
+
+    @Override
+    public DataPosition mark()
+    {
+        resetChecksum();
+        return delegate().mark();
+    }
+
+    @Override
+    public void reset(DataPosition mark) throws IOException
+    {
+        resetChecksum();
+        delegate().reset(mark);
+    }
+
+    @Override
+    public long bytesPastMark(DataPosition mark)
+    {
+        return delegate().bytesPastMark(mark);
+    }
+}

src/java/org/apache/cassandra/io/util/ChecksumedRandomAccessReader.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.util.function.Supplier;
+import java.util.zip.Checksum;
+
+public class ChecksumedRandomAccessReader extends ChecksumedFileDataInput
+{
+    public ChecksumedRandomAccessReader(RandomAccessReader delegate, Checksum checksum)
+    {
+        super(delegate, checksum);
+    }
+
+    public ChecksumedRandomAccessReader(RandomAccessReader delegate, Supplier<Checksum> fn)
+    {
+        super(delegate, fn);
+    }
+
+    @Override
+    public RandomAccessReader delegate()
+    {
+        return (RandomAccessReader) super.delegate();
+    }
+
+    public long length()
+    {
+        return delegate().length();
+    }
+}

src/java/org/apache/cassandra/io/util/ChecksumedSequentialWriter.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+import java.util.zip.Checksum;
+
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+public class ChecksumedSequentialWriter extends ChecksumedDataOutputPlus implements Transactional
+{
+    private static final SequentialWriterOption DEFAULT_WRITER_OPTIONS = SequentialWriterOption.newBuilder().finishOnClose(true).build();
+
+    public ChecksumedSequentialWriter(SequentialWriter delegate, Checksum checksum)
+    {
+        super(delegate, checksum);
+    }
+
+    public ChecksumedSequentialWriter(SequentialWriter delegate, Supplier<Checksum> fn)
+    {
+        super(delegate, fn);
+    }
+
+    public static ChecksumedSequentialWriter open(File file, boolean append, Supplier<Checksum> fn) throws IOException
+    {
+        SequentialWriter writer = new SequentialWriter(file, DEFAULT_WRITER_OPTIONS);
+        if (append)
+            writer.skipBytes(file.length());
+        return new ChecksumedSequentialWriter(writer, fn);
+    }
+
+    @Override
+    public SequentialWriter delegate()
+    {
+        return (SequentialWriter) super.delegate();
+    }
+
+    public long getFilePointer()
+    {
+        return delegate().position();
+    }
+
+    @Override
+    public Throwable commit(Throwable accumulate)
+    {
+        return delegate().commit(accumulate);
+    }
+
+    @Override
+    public Throwable abort(Throwable accumulate)
+    {
+        return delegate().abort(accumulate);
+    }
+
+    @Override
+    public void prepareToCommit()
+    {
+        delegate().prepareToCommit();
+    }
+
+    @Override
+    public void close()
+    {
+        delegate().close();
+    }
+}

src/java/org/apache/cassandra/io/util/DataInputPlus.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.io.util;
 
 import java.io.DataInput;
+import java.io.DataInputStream;
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -98,6 +99,81 @@ public interface DataInputPlus extends DataInput
             throw new EOFException("EOF after " + skipped + " bytes out of " + n);
     }
 
+    @Override
+    default byte readByte() throws IOException
+    {
+        return (byte) readUnsignedByte();
+    }
+
+    @Override
+    default boolean readBoolean() throws IOException
+    {
+        return readUnsignedByte() != 0;
+    }
+
+    @Override
+    default short readShort() throws IOException
+    {
+        int ch1 = readUnsignedByte();
+        int ch2 = readUnsignedByte();
+        if ((ch1 | ch2) < 0)
+            throw new EOFException();
+        return (short)((ch1 << 8) + (ch2 << 0));
+    }
+
+    @Override
+    default int readUnsignedShort() throws IOException
+    {
+        int ch1 = readUnsignedByte();
+        int ch2 = readUnsignedByte();
+        if ((ch1 | ch2) < 0)
+            throw new EOFException();
+        return (ch1 << 8) + (ch2 << 0);
+    }
+
+    @Override
+    default char readChar() throws IOException
+    {
+        int ch1 = readUnsignedByte();
+        int ch2 = readUnsignedByte();
+        if ((ch1 | ch2) < 0)
+            throw new EOFException();
+        return (char)((ch1 << 8) + (ch2 << 0));
+    }
+
+    @Override
+    default int readInt() throws IOException
+    {
+        int ch1 = readUnsignedByte();
+        int ch2 = readUnsignedByte();
+        int ch3 = readUnsignedByte();
+        int ch4 = readUnsignedByte();
+        if ((ch1 | ch2 | ch3 | ch4) < 0)
+            throw new EOFException();
+        return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
+    }
+
+    @Override
+    long readLong() throws IOException;
+
+    @Override
+    default float readFloat() throws IOException
+    {
+        return Float.intBitsToFloat(readInt());
+    }
+
+    @Override
+    default double readDouble() throws IOException
+    {
+        return Double.longBitsToDouble(readLong());
+    }
+
+    @Override
+    default String readUTF() throws IOException
+    {
+        return DataInputStream.readUTF(this);
+    }
+
     /**
      * Wrapper around an InputStream that provides no buffering but can decode varints
      */

src/java/org/apache/cassandra/io/util/DataOutputPlus.java

@@ -193,4 +193,65 @@ public interface DataOutputPlus extends DataOutput
     {
         throw new UnsupportedOperationException();
     }
+
+    @Override
+    default void writeBoolean(boolean v) throws IOException
+    {
+        write(v ? 1 : 0);
+    }
+
+    @Override
+    default void writeShort(int v) throws IOException
+    {
+        write((v >>> 8) & 0xFF);
+        write((v >>> 0) & 0xFF);
+    }
+
+    @Override
+    default void writeChar(int v) throws IOException
+    {
+        write((v >>> 8) & 0xFF);
+        write((v >>> 0) & 0xFF);
+    }
+
+    @Override
+    default void writeInt(int v) throws IOException
+    {
+        write((v >>> 24) & 0xFF);
+        write((v >>> 16) & 0xFF);
+        write((v >>>  8) & 0xFF);
+        write((v >>>  0) & 0xFF);
+    }
+
+    @Override
+    default void writeFloat(float v) throws IOException
+    {
+        writeInt(Float.floatToIntBits(v));
+    }
+
+    @Override
+    default void writeDouble(double v) throws IOException
+    {
+        writeLong(Double.doubleToLongBits(v));
+    }
+
+    @Override
+    default void writeBytes(String s) throws IOException
+    {
+        int len = s.length();
+        for (int i = 0 ; i < len ; i++) {
+            write((byte)s.charAt(i));
+        }
+    }
+
+    @Override
+    default void writeChars(String s) throws IOException
+    {
+        int len = s.length();
+        for (int i = 0 ; i < len ; i++) {
+            int v = s.charAt(i);
+            write((v >>> 8) & 0xFF);
+            write((v >>> 0) & 0xFF);
+        }
+    }
 }

src/java/org/apache/cassandra/journal/Journal.java

@@ -195,7 +195,10 @@ public class Journal<K, V> implements Shutdownable
     @Override
     public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
     {
-        return false;
+        boolean r = true;
+        r &= allocator.awaitTermination(timeout, units);
+        r &= closer.awaitTermination(timeout, units);
+        return r;
     }
 
     /**

src/java/org/apache/cassandra/locator/ReplicaLayout.java

@@ -24,6 +24,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.TableId;
@@ -405,6 +407,10 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
 
     static EndpointsForToken forLocalStrategyToken(ClusterMetadata metadata, AbstractReplicationStrategy replicationStrategy, Token t)
     {
-        return replicationStrategy.calculateNaturalReplicas(t, metadata).forToken(t);
+        if (!(t instanceof LocalPartitioner.LocalToken))
+            return replicationStrategy.calculateNaturalReplicas(t, metadata).forToken(t);
+
+        // local tokens use a different partitioner than the global one... so update the ranges
+        return EndpointsForToken.of(t, new Replica(FBUtilities.getBroadcastAddressAndPort(), new Range<>(t, t), true));
     }
 }

src/java/org/apache/cassandra/schema/SchemaProvider.java

@@ -35,6 +35,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.KeyspaceNotDefinedException;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.UnknownTableException;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -135,6 +136,13 @@ public interface SchemaProvider
     @Nullable
     TableMetadata getTableMetadata(TableId id);
 
+    @Nullable
+    default IPartitioner getTablePartitioner(TableId id)
+    {
+        TableMetadata metadata = getTableMetadata(id);
+        return metadata == null ? null : metadata.partitioner;
+    }
+
     @Nullable
     default TableMetadataRef getTableMetadataRef(TableId id)
     {

src/java/org/apache/cassandra/schema/TableId.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.Pair;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -50,7 +51,8 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
 public class TableId implements Comparable<TableId>
 {
     public static final long MAGIC = 1956074401491665062L;
-    // TODO: should this be a TimeUUID?
+    public static final long EMPTY_SIZE = ObjectSizes.measureDeep(new UUID(0, 0));
+
     private final UUID id;
 
     private TableId(UUID id)

src/java/org/apache/cassandra/service/accord/AccordCachingState.java

@@ -121,6 +121,11 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
         return status().isComplete();
     }
 
+    int lastQueriedEstimatedSizeOnHeap()
+    {
+        return lastQueriedEstimatedSizeOnHeap;
+    }
+
     int estimatedSizeOnHeap(ToLongFunction<V> estimator)
     {
         shouldUpdateSize = false;   // TODO (expected): probably not the safest place to clear need to compute size
@@ -600,6 +605,12 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
             return isDone();
         }
 
+        @Override
+        public V get()
+        {
+            return current;
+        }
+
         @Override
         public State<K, V> complete()
         {

src/java/org/apache/cassandra/service/accord/AccordCommandStore.java

@@ -18,8 +18,7 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.Collections;
-import java.util.List;
+import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Objects;
@@ -27,23 +26,19 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
-import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
-
 import javax.annotation.Nullable;
 
+import accord.api.Key;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.api.Agent;
 import accord.api.DataStore;
-import accord.api.Key;
 import accord.api.ProgressLog;
 import accord.local.CommandsForKey;
-import accord.impl.CommandsSummary;
 import accord.impl.TimestampsForKey;
 import accord.local.Command;
 import accord.local.CommandStore;
@@ -52,37 +47,28 @@ import accord.local.NodeTimeService;
 import accord.local.PreLoadContext;
 import accord.local.RedundantBefore;
 import accord.local.SafeCommandStore;
-import accord.local.SaveStatus;
-import accord.local.SerializerSupport;
 import accord.local.SerializerSupport.MessageProvider;
 import accord.messages.Message;
-import accord.primitives.AbstractKeys;
-import accord.primitives.AbstractRanges;
-import accord.primitives.Ballot;
-import accord.primitives.Range;
 import accord.primitives.Ranges;
-import accord.primitives.Routable;
 import accord.primitives.RoutableKey;
-import accord.primitives.Routables;
-import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.ReducingRangeMap;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
-import accord.utils.async.Observable;
 import org.apache.cassandra.cache.CacheSize;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.metrics.AccordStateCacheMetrics;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.async.AsyncOperation;
 import org.apache.cassandra.service.accord.async.ExecutionOrder;
+import org.apache.cassandra.service.accord.events.CacheEvents;
 import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@@ -91,8 +77,12 @@ public class AccordCommandStore extends CommandStore implements CacheSize
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordCommandStore.class);
 
+    private static final boolean CHECK_THREADS = CassandraRelevantProperties.TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED.getBoolean();
+
     private static long getThreadId(ExecutorService executor)
     {
+        if (!CHECK_THREADS)
+            return 0;
         try
         {
             return executor.submit(() -> Thread.currentThread().getId()).get();
@@ -109,7 +99,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
 
     private final long threadId;
     public final String loggingId;
-    private final AccordJournal journal;
+    private final IJournal journal;
     private final ExecutorService executor;
     private final ExecutionOrder executionOrder;
     private final AccordStateCache stateCache;
@@ -119,7 +109,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     private AsyncOperation<?> currentOperation = null;
     private AccordSafeCommandStore current = null;
     private long lastSystemTimestampMicros = Long.MIN_VALUE;
-    private CommandsForRanges commandsForRanges = new CommandsForRanges();
+    private final CommandsForRangesLoader commandsForRangesLoader;
 
     public AccordCommandStore(int id,
                               NodeTimeService time,
@@ -127,12 +117,79 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                               DataStore dataStore,
                               ProgressLog.Factory progressLogFactory,
                               EpochUpdateHolder epochUpdateHolder,
-                              AccordJournal journal,
+                              IJournal journal,
                               AccordStateCacheMetrics cacheMetrics)
     {
         this(id, time, agent, dataStore, progressLogFactory, epochUpdateHolder, journal, Stage.READ.executor(), Stage.MUTATION.executor(), cacheMetrics);
     }
 
+    private static <K, V> void registerJfrListener(int id, AccordStateCache.Instance<K, V, ?> instance, String name)
+    {
+        if (!DatabaseDescriptor.getAccordStateCacheListenerJFREnabled())
+            return;
+        instance.register(new AccordStateCache.Listener<K, V>() {
+            private final IdentityHashMap<AccordCachingState<?, ?>, CacheEvents.Evict> pendingEvicts = new IdentityHashMap<>();
+
+            @Override
+            public void onAdd(AccordCachingState<K, V> state)
+            {
+                CacheEvents.Add add = new CacheEvents.Add();
+                CacheEvents.Evict evict = new CacheEvents.Evict();
+                if (!add.isEnabled())
+                    return;
+                add.begin();
+                evict.begin();
+                add.store = evict.store = id;
+                add.instance = evict.instance = name;
+                add.key = evict.key = state.key().toString();
+                updateMutable(instance, state, add);
+                add.commit();
+                pendingEvicts.put(state, evict);
+            }
+
+            @Override
+            public void onRelease(AccordCachingState<K, V> state)
+            {
+
+            }
+
+            @Override
+            public void onEvict(AccordCachingState<K, V> state)
+            {
+                CacheEvents.Evict event = pendingEvicts.remove(state);
+                if (event == null) return;
+                updateMutable(instance, state, event);
+                event.commit();
+            }
+        });
+    }
+
+    private static void updateMutable(AccordStateCache.Instance<?, ?, ?> instance, AccordCachingState<?, ?> state, CacheEvents event)
+    {
+        event.status = state.state().status().name();
+
+        event.lastQueriedEstimatedSizeOnHeap = state.lastQueriedEstimatedSizeOnHeap();
+
+        event.instanceAllocated = instance.weightedSize();
+        AccordStateCache.Stats stats = instance.stats();
+        event.instanceStatsQueries = stats.queries;
+        event.instanceStatsHits = stats.hits;
+        event.instanceStatsMisses = stats.misses;
+
+        event.globalSize = instance.size();
+        event.globalReferenced = instance.globalReferencedEntries();
+        event.globalUnreferenced = instance.globalUnreferencedEntries();
+        event.globalCapacity = instance.capacity();
+        event.globalAllocated = instance.globalAllocated();
+
+        stats = instance.globalStats();
+        event.globalStatsQueries = stats.queries;
+        event.globalStatsHits = stats.hits;
+        event.globalStatsMisses = stats.misses;
+
+        event.update();
+    }
+
     @VisibleForTesting
     public AccordCommandStore(int id,
                               NodeTimeService time,
@@ -140,7 +197,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                               DataStore dataStore,
                               ProgressLog.Factory progressLogFactory,
                               EpochUpdateHolder epochUpdateHolder,
-                              AccordJournal journal,
+                              IJournal journal,
                               ExecutorPlus loadExecutor,
                               ExecutorPlus saveExecutor,
                               AccordStateCacheMetrics cacheMetrics)
@@ -160,6 +217,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                                 this::saveCommand,
                                 this::validateCommand,
                                 AccordObjectSizes::command);
+        registerJfrListener(id, commandCache, "Command");
         timestampsForKeyCache =
             stateCache.instance(Key.class,
                                 AccordSafeTimestampsForKey.class,
@@ -168,6 +226,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                                 this::saveTimestampsForKey,
                                 this::validateTimestampsForKey,
                                 AccordObjectSizes::timestampsForKey);
+        registerJfrListener(id, timestampsForKeyCache, "TimestampsForKey");
         commandsForKeyCache =
             stateCache.instance(Key.class,
                                 AccordSafeCommandsForKey.class,
@@ -177,6 +236,9 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                                 this::validateCommandsForKey,
                                 AccordObjectSizes::commandsForKey,
                                 AccordCachingState::new);
+        registerJfrListener(id, commandsForKeyCache, "CommandsForKey");
+
+        this.commandsForRangesLoader = new CommandsForRangesLoader(this);
 
         AccordKeyspace.loadCommandStoreMetadata(id, ((rejectBefore, durableBefore, redundantBefore, bootstrapBeganAt, safeToRead) -> {
             executor.submit(() -> {
@@ -194,7 +256,6 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         }));
 
         executor.execute(() -> CommandStore.register(this));
-        executor.execute(this::loadRangesToCommands);
     }
 
     static Factory factory(AccordJournal journal, AccordStateCacheMetrics cacheMetrics)
@@ -203,66 +264,16 @@ public class AccordCommandStore extends CommandStore implements CacheSize
                new AccordCommandStore(id, time, agent, dataStore, progressLogFactory, rangesForEpoch, journal, cacheMetrics);
     }
 
-    private void loadRangesToCommands()
+    public CommandsForRangesLoader diskCommandsForRanges()
     {
-        AsyncPromise<CommandsForRanges> future = new AsyncPromise<>();
-        AccordKeyspace.findAllCommandsByDomain(id, Routable.Domain.Range, ImmutableSet.of("txn_id", "status", "accepted_ballot", "execute_at"), new Observable<UntypedResultSet.Row>()
-        {
-            private CommandsForRanges.Builder builder = new CommandsForRanges.Builder();
-
-            @Override
-            public void onNext(UntypedResultSet.Row row) throws Exception
-            {
-                TxnId txnId = AccordKeyspace.deserializeTxnId(row);
-                SaveStatus status = AccordKeyspace.deserializeStatus(row);
-                Timestamp executeAt = AccordKeyspace.deserializeExecuteAtOrNull(row);
-                Ballot accepted = AccordKeyspace.deserializeAcceptedOrNull(row);
-
-                MessageProvider messageProvider = journal.makeMessageProvider(txnId);
-
-                SerializerSupport.TxnAndDeps txnAndDeps = SerializerSupport.extractTxnAndDeps(unsafeRangesForEpoch(), status, accepted, messageProvider);
-                Seekables<?, ?> keys = txnAndDeps.txn.keys();
-                if (keys.domain() != Routable.Domain.Range)
-                    throw new AssertionError(String.format("Txn keys are not range for %s", txnAndDeps.txn));
-                Ranges ranges = (Ranges) keys;
-
-                List<TxnId> dependsOn = txnAndDeps.deps == null ? Collections.emptyList() : txnAndDeps.deps.txnIds();
-                builder.put(txnId, ranges, status, executeAt, dependsOn);
-            }
-
-            @Override
-            public void onError(Throwable t)
-            {
-                builder = null;
-                future.tryFailure(t);
-            }
-
-            @Override
-            public void onCompleted()
-            {
-                CommandsForRanges result = this.builder.build();
-                builder = null;
-                future.trySuccess(result);
-            }
-        });
-        try
-        {
-            commandsForRanges = future.get();
-            logger.debug("Loaded {} intervals", commandsForRanges.size());
-        }
-        catch (InterruptedException e)
-        {
-            throw new UncheckedInterruptedException(e);
-        }
-        catch (ExecutionException e)
-        {
-            throw new RuntimeException(e.getCause());
-        }
+        return commandsForRangesLoader;
     }
 
     @Override
     public boolean inStore()
     {
+        if (!CHECK_THREADS)
+            return true;
         return Thread.currentThread().getId() == threadId;
     }
 
@@ -298,6 +309,8 @@ public class AccordCommandStore extends CommandStore implements CacheSize
 
     public void checkNotInStoreThread()
     {
+        if (!CHECK_THREADS)
+            return;
         Invariants.checkState(!inStore());
     }
 
@@ -320,7 +333,6 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     {
         return commandsForKeyCache;
     }
-
     Command loadCommand(TxnId txnId)
     {
         return AccordKeyspace.loadCommand(this, txnId);
@@ -467,13 +479,16 @@ public class AccordCommandStore extends CommandStore implements CacheSize
     public AccordSafeCommandStore beginOperation(PreLoadContext preLoadContext,
                                                  Map<TxnId, AccordSafeCommand> commands,
                                                  NavigableMap<Key, AccordSafeTimestampsForKey> timestampsForKeys,
-                                                 NavigableMap<Key, AccordSafeCommandsForKey> commandsForKeys)
+                                                 NavigableMap<Key, AccordSafeCommandsForKey> commandsForKeys,
+                                                 @Nullable AccordSafeCommandsForRanges commandsForRanges)
     {
         Invariants.checkState(current == null);
         commands.values().forEach(AccordSafeState::preExecute);
         commandsForKeys.values().forEach(AccordSafeState::preExecute);
         timestampsForKeys.values().forEach(AccordSafeState::preExecute);
-        current = new AccordSafeCommandStore(preLoadContext, commands, timestampsForKeys, commandsForKeys, this);
+        if (commandsForRanges != null)
+            commandsForRanges.preExecute();
+        current = new AccordSafeCommandStore(preLoadContext, commands, timestampsForKeys, commandsForKeys, commandsForRanges, this);
         return current;
     }
 
@@ -489,46 +504,6 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         current = null;
     }
 
-    <O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
-    {
-        keysOrRanges = keysOrRanges.slice(slice, Routables.Slice.Minimal);
-        switch (keysOrRanges.domain())
-        {
-            case Key:
-            {
-                AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
-                for (CommandsSummary summary : commandsForRanges.search(keys))
-                    accumulate = map.apply(summary, accumulate);
-            }
-            break;
-            case Range:
-            {
-                AbstractRanges ranges = (AbstractRanges) keysOrRanges;
-                for (Range range : ranges)
-                {
-                    CommandsSummary summary = commandsForRanges.search(range);
-                    if (summary == null)
-                        continue;
-                    accumulate = map.apply(summary, accumulate);
-                }
-            }
-            break;
-            default:
-                throw new AssertionError("Unknown domain: " + keysOrRanges.domain());
-        }
-        return accumulate;
-    }
-
-    CommandsForRanges commandsForRanges()
-    {
-        return commandsForRanges;
-    }
-
-    CommandsForRanges.Updater updateRanges()
-    {
-        return commandsForRanges.update();
-    }
-
     public void abortCurrentOperation()
     {
         current = null;
@@ -576,13 +551,6 @@ public class AccordCommandStore extends CommandStore implements CacheSize
         AccordKeyspace.updateRedundantBefore(this, newRedundantBefore);
     }
 
-    @Override
-    public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
-    {
-        super.markShardDurable(safeStore, globalSyncId, ranges);
-        commandsForRanges.prune(globalSyncId, ranges);
-    }
-
     public NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return super.bootstrapBeganAt(); }
     public NavigableMap<Timestamp, Ranges> safeToRead() { return super.safeToRead(); }
 

src/java/org/apache/cassandra/service/accord/AccordJournal.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -27,6 +28,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 import java.util.function.Predicate;
 import java.util.zip.Checksum;
@@ -99,6 +101,7 @@ import org.apache.cassandra.service.accord.serializers.PreacceptSerializers;
 import org.apache.cassandra.service.accord.serializers.RecoverySerializers;
 import org.apache.cassandra.service.accord.serializers.SetDurableSerializers;
 import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.ExecutorUtils;
 import org.apache.cassandra.utils.concurrent.Semaphore;
 import org.jctools.queues.SpscLinkedQueue;
 
@@ -143,7 +146,7 @@ import static org.apache.cassandra.utils.CollectionSerializers.serializedListSiz
 import static org.apache.cassandra.utils.concurrent.Semaphore.newSemaphore;
 import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize;
 
-public class AccordJournal implements Shutdownable
+public class AccordJournal implements IJournal, Shutdownable
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordJournal.class);
 
@@ -262,8 +265,15 @@ public class AccordJournal implements Shutdownable
     @Override
     public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
     {
-        // TODO (expected, other)
-        return true;
+        try
+        {
+            ExecutorUtils.awaitTermination(timeout, units, Arrays.asList(journal, frameAggregator, frameApplicator));
+            return true;
+        }
+        catch (TimeoutException e)
+        {
+            return false;
+        }
     }
 
     /**
@@ -296,6 +306,7 @@ public class AccordJournal implements Shutdownable
     }
 
     @VisibleForTesting
+    @Override
     public void appendMessageBlocking(Message message)
     {
         Type type = Type.fromMessageType(message.type());
@@ -1061,7 +1072,7 @@ public class AccordJournal implements Shutdownable
      * Once written, the frame record is submitted to {@link FrameApplicator}, which will process all the framed
      * requests once the frame has been flushed to disk.
      */
-    private final class FrameAggregator implements Interruptible.Task
+    private final class FrameAggregator implements Interruptible.Task, Shutdownable
     {
         /* external MPSC pending request queue */
         private final ManyToOneConcurrentLinkedQueue<RequestContext> unframedRequests = new ManyToOneConcurrentLinkedQueue<>();
@@ -1091,9 +1102,26 @@ public class AccordJournal implements Shutdownable
             executor = executorFactory().infiniteLoop("AccordJournal#FrameAggregator", this, SAFE, NON_DAEMON, SYNCHRONIZED);
         }
 
-        void shutdown()
+        @Override
+        public boolean isTerminated() {
+            return executor == null || executor.isTerminated();
+        }
+
+        @Override
+        public void shutdown()
         {
-            executor.shutdown();
+            if (executor != null)
+                executor.shutdown();
+        }
+
+        @Override
+        public Object shutdownNow() {
+            return executor == null ? null : executor.shutdownNow();
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException {
+            return executor == null || executor.awaitTermination(timeout, units);
         }
 
         @Override
@@ -1168,7 +1196,7 @@ public class AccordJournal implements Shutdownable
      * Gets the aggregated frames containing previously written requests/messages,
      * and sorts and "applies" them once part of the journal that fully contains them is flushed.
      */
-    private final class FrameApplicator implements Runnable
+    private final class FrameApplicator implements Runnable, Shutdownable
     {
         /** external SPSC written frame queue */
         private final SpscLinkedQueue<PendingFrame> newFrames = new SpscLinkedQueue<>();
@@ -1199,9 +1227,26 @@ public class AccordJournal implements Shutdownable
             executor = executorFactory().sequential("AccordJournal#FrameApplicator");
         }
 
-        void shutdown()
+        @Override
+        public boolean isTerminated() {
+            return executor == null || executor.isTerminated();
+        }
+
+        @Override
+        public void shutdown()
         {
-            executor.shutdown();
+            if (executor != null)
+                executor.shutdown();
+        }
+
+        @Override
+        public Object shutdownNow() {
+            return executor == null ? null : executor.shutdownNow();
+        }
+
+        @Override
+        public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException {
+            return executor == null || executor.awaitTermination(timeout, units);
         }
 
         @Override
@@ -1334,8 +1379,8 @@ public class AccordJournal implements Shutdownable
     /*
      * Message provider implementation
      */
-
-    SerializerSupport.MessageProvider makeMessageProvider(TxnId txnId)
+    @Override
+    public SerializerSupport.MessageProvider makeMessageProvider(TxnId txnId)
     {
         return LOG_MESSAGE_PROVIDER ? new LoggingMessageProvider(txnId, new MessageProvider(txnId)) : new MessageProvider(txnId);
     }

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -27,7 +27,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
@@ -37,6 +40,7 @@ import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -118,17 +122,23 @@ import org.apache.cassandra.db.rows.Row.Deletion;
 import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.db.transform.FilteredPartitions;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.LocalPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.accord.RouteIndex;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.LocalVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.Tables;
@@ -138,7 +148,9 @@ import org.apache.cassandra.schema.Views;
 import org.apache.cassandra.serializers.UUIDSerializer;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers;
 import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
@@ -192,6 +204,15 @@ public class AccordKeyspace
 
     private static final ClusteringIndexFilter FULL_PARTITION = new ClusteringIndexSliceFilter(Slices.ALL, false);
 
+    //TODO (now, performance): should this be partitioner rather than TableId?  As of this patch distributed tables should only have 1 partitioner...
+    private static final ConcurrentMap<TableId, AccordRoutingKeyByteSource.Serializer> TABLE_SERIALIZERS = new ConcurrentHashMap<>();
+
+    // Schema needs all system keyspace, and this is a system keyspace!  So can not touch schema in init
+    private static class SchemaHolder
+    {
+        private static SchemaProvider schema = Objects.requireNonNull(Schema.instance);
+    }
+
     private enum TokenType
     {
         Murmur3((byte) 1),
@@ -216,7 +237,7 @@ public class AccordKeyspace
     }
 
     // TODO: store timestamps as blobs (confirm there are no negative numbers, or offset)
-    private static final TableMetadata Commands =
+    public static final TableMetadata Commands =
         parse(COMMANDS,
               "accord commands",
               "CREATE TABLE %s ("
@@ -234,10 +255,13 @@ public class AccordKeyspace
               + "PRIMARY KEY((store_id, domain, txn_id))"
               + ')')
         .partitioner(new LocalPartitioner(CompositeType.getInstance(Int32Type.instance, Int32Type.instance, TIMESTAMP_TYPE)))
+        .indexes(Indexes.builder()
+                        .add(IndexMetadata.fromSchemaMetadata("route", IndexMetadata.Kind.CUSTOM, ImmutableMap.of("class_name", RouteIndex.class.getCanonicalName(), "target", "route")))
+                        .build())
         .build();
 
     // TODO: naming is not very clearly distinct from the base serializers
-    private static class LocalVersionedSerializers
+    public static class LocalVersionedSerializers
     {
         static final LocalVersionedSerializer<Route<?>> route = localSerializer(KeySerializers.route);
         static final LocalVersionedSerializer<Command.DurableAndIdempotentListener> listeners = localSerializer(ListenerSerializers.listener);
@@ -266,8 +290,8 @@ public class AccordKeyspace
     {
         static final ClusteringComparator keyComparator = Commands.partitionKeyAsClusteringComparator();
         static final CompositeType partitionKeyType = (CompositeType) Commands.partitionKeyType;
-        static final ColumnMetadata txn_id = getColumn(Commands, "txn_id");
-        static final ColumnMetadata store_id = getColumn(Commands, "store_id");
+        public static final ColumnMetadata txn_id = getColumn(Commands, "txn_id");
+        public static final ColumnMetadata store_id = getColumn(Commands, "store_id");
         public static final ColumnMetadata status = getColumn(Commands, "status");
         public static final ColumnMetadata route = getColumn(Commands, "route");
         public static final ColumnMetadata durability = getColumn(Commands, "durability");
@@ -298,6 +322,12 @@ public class AccordKeyspace
             return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
         }
 
+        public static int getStoreId(DecoratedKey pk)
+        {
+            ByteBuffer[] array = splitPartitionKey(pk);
+            return getStoreId(array);
+        }
+
         public static TxnId getTxnId(ByteBuffer[] partitionKeyComponents)
         {
             return deserializeTimestampOrNull(partitionKeyComponents[txn_id.position()], ByteBufferAccessor.instance, TxnId::fromBits);
@@ -409,6 +439,8 @@ public class AccordKeyspace
         }
     }
 
+    //TODO (now, performance): do we actually care about the sort ordering?  We don't do range scans on this table
+    //TODO (now, performance): should we remove key_token?  We don't need it so its just added space
     private static final TableMetadata TimestampsForKeys =
         parse(TIMESTAMPS_FOR_KEY,
               "accord timestamps per key",
@@ -578,6 +610,11 @@ public class AccordKeyspace
             return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
         }
 
+        public PartitionKey getKey(DecoratedKey key)
+        {
+            return getKey(splitPartitionKey(key));
+        }
+
         public PartitionKey getKey(ByteBuffer[] partitionKeyComponents)
         {
             return deserializeKey(partitionKeyComponents[key.position()]);
@@ -860,12 +897,13 @@ public class AccordKeyspace
         return value;
     }
 
-    private static ByteBuffer serializeKey(PartitionKey key)
+    @VisibleForTesting
+    public static ByteBuffer serializeKey(PartitionKey key)
     {
         return KEY_TYPE.pack(UUIDSerializer.instance.serialize(key.table().asUUID()), key.partitionKey().getKey());
     }
 
-    private static ByteBuffer serializeTimestamp(Timestamp timestamp)
+    public static ByteBuffer serializeTimestamp(Timestamp timestamp)
     {
         return TIMESTAMP_TYPE.pack(bytes(timestamp.msb), bytes(timestamp.lsb), bytes(timestamp.node.id));
     }
@@ -1097,15 +1135,15 @@ public class AccordKeyspace
     }
 
     public static void findAllKeysBetween(int commandStore,
-                                          Token start, boolean startInclusive,
-                                          Token end, boolean endInclusive,
+                                          AccordRoutingKey start, boolean startInclusive,
+                                          AccordRoutingKey end, boolean endInclusive,
                                           Observable<PartitionKey> callback)
     {
         //TODO (optimize) : CQL doesn't look smart enough to only walk Index.db, and ends up walking the Data.db file for each row in the partitions found (for frequent keys, this cost adds up)
         // it would be possible to find all SSTables that "could" intersect this range, then have a merge iterator over the Index.db (filtered to the range; index stores partition liveness)...
         KeysBetween work = new KeysBetween(commandStore,
-                                           AccordKeyspace.serializeToken(start), startInclusive,
-                                           AccordKeyspace.serializeToken(end), endInclusive,
+                                           AccordKeyspace.serializeRoutingKey(start), startInclusive,
+                                           AccordKeyspace.serializeRoutingKey(end), endInclusive,
                                            ImmutableSet.of("key"),
                                            Stage.READ.executor(), Observable.distinct(callback).map(AccordKeyspace::deserializeKey));
         work.schedule();
@@ -1131,14 +1169,14 @@ public class AccordKeyspace
             this.start = start;
             this.end = end;
 
-            String selection = selection(TimestampsForKeys, requiredColumns, COLUMNS_FOR_ITERATION);
+            String selection = selection(CommandsForKeys, requiredColumns, COLUMNS_FOR_ITERATION);
             this.cqlFirst = format("SELECT DISTINCT %s\n" +
                                           "FROM %s\n" +
                                           "WHERE store_id = ?\n" +
                                           (startInclusive ? "  AND key_token >= ?\n" : "  AND key_token > ?\n") +
                                           (endInclusive ? "  AND key_token <= ?\n" : "  AND key_token < ?\n") +
                                           "ALLOW FILTERING",
-                                          selection, TimestampsForKeys);
+                                          selection, CommandsForKeys);
             this.cqlContinue = format("SELECT DISTINCT %s\n" +
                                              "FROM %s\n" +
                                              "WHERE store_id = ?\n" +
@@ -1146,7 +1184,7 @@ public class AccordKeyspace
                                              "  AND key > ?\n" +
                                              (endInclusive ? "  AND key_token <= ?\n" : "  AND key_token < ?\n") +
                                              "ALLOW FILTERING",
-                                             selection, TimestampsForKeys);
+                                             selection, CommandsForKeys);
         }
 
         @Override
@@ -1216,11 +1254,16 @@ public class AccordKeyspace
         return Status.Durability.values()[row.getInt("durability", 0)];
     }
 
-    private static Route<?> deserializeRouteOrNull(ByteBuffer bytes) throws IOException
+    public static Route<?> deserializeRouteOrNull(ByteBuffer bytes) throws IOException
     {
         return bytes != null && !ByteBufferAccessor.instance.isEmpty(bytes) ? deserialize(bytes, LocalVersionedSerializers.route) : null;
     }
 
+    public static ByteBuffer serializeRoute(Route<?> route) throws IOException
+    {
+        return serialize(route, LocalVersionedSerializers.route);
+    }
+
     private static Route<?> deserializeRouteOrNull(UntypedResultSet.Row row) throws IOException
     {
         return deserializeRouteOrNull(row.getBlob("route"));
@@ -1303,10 +1346,10 @@ public class AccordKeyspace
         TableId tableId = TableId.fromUUID(UUIDSerializer.instance.deserialize(split.get(0)));
         ByteBuffer key = split.get(1);
 
-        TableMetadata metadata = Schema.instance.getTableMetadata(tableId);
-        if (metadata == null)
+        IPartitioner partitioner = SchemaHolder.schema.getTablePartitioner(tableId);
+        if (partitioner == null)
             throw new IllegalStateException("Table with id " + tableId + " could not be found; was it deleted?");
-        return new PartitionKey(tableId, metadata.partitioner.decorateKey(key));
+        return new PartitionKey(tableId, partitioner.decorateKey(key));
     }
 
     public static PartitionKey deserializeKey(UntypedResultSet.Row row)
@@ -1390,16 +1433,36 @@ public class AccordKeyspace
 
     private static DecoratedKey makeKey(CommandsForKeyAccessor accessor, int storeId, PartitionKey key)
     {
-        Token token = key.token();
         ByteBuffer pk = accessor.keyComparator.make(storeId,
-                                                    serializeToken(token),
+                                                    serializeRoutingKey(key.toUnseekable()),
                                                     serializeKey(key)).serializeAsPartitionKey();
         return accessor.table.partitioner.decorateKey(pk);
     }
 
+    @VisibleForTesting
+    public static ByteBuffer serializeRoutingKey(AccordRoutingKey routingKey)
+    {
+        AccordRoutingKeyByteSource.Serializer serializer = TABLE_SERIALIZERS.computeIfAbsent(routingKey.table(), ignore -> {
+            IPartitioner partitioner;
+            if (routingKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.TOKEN)
+                partitioner = routingKey.asTokenKey().token().getPartitioner();
+            else
+                partitioner = SchemaHolder.schema.getTablePartitioner(routingKey.table());
+            return AccordRoutingKeyByteSource.variableLength(partitioner);
+        });
+        byte[] bytes = serializer.serialize(routingKey);
+        return ByteBuffer.wrap(bytes);
+    }
+
     private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, PartitionKey key, CommandsForKey commandsForKey, long timestampMicros)
     {
         ByteBuffer bytes = CommandsForKeySerializer.toBytesWithoutKey(commandsForKey);
+        return getCommandsForKeyPartitionUpdate(storeId, key, timestampMicros, bytes);
+    }
+
+    @VisibleForTesting
+    public static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, PartitionKey key, long timestampMicros, ByteBuffer bytes)
+    {
         return singleRowUpdate(CommandsForKeysAccessor.table,
                                makeKey(CommandsForKeysAccessor, storeId, key),
                                singleCellRow(Clustering.EMPTY, BufferCell.live(CommandsForKeysAccessor.data, timestampMicros, bytes)));
@@ -1832,4 +1895,19 @@ public class AccordKeyspace
         consumer.accept(rejectBefore, durableBefore, redundantBefore, bootstrapBeganAt, safeToRead);
     }
 
+    @VisibleForTesting
+    public static void unsafeSetSchema(SchemaProvider provider)
+    {
+        SchemaHolder.schema = provider;
+    }
+
+    @VisibleForTesting
+    public static void unsafeClear()
+    {
+        for (ColumnFamilyStore store : Keyspace.open(SchemaConstants.ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+            store.truncateBlockingWithoutSnapshot();
+        TABLE_SERIALIZERS.clear();
+        SchemaHolder.schema = Schema.instance;
+    }
+
 }

src/java/org/apache/cassandra/service/accord/AccordMessageSink.java

@@ -222,7 +222,7 @@ public class AccordMessageSink implements MessageSink
         Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
         Message<Request> message = Message.out(verb, request);
         InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
-        logger.debug("Sending {} {} to {}", verb, message.payload, endpoint);
+        logger.trace("Sending {} {} to {}", verb, message.payload, endpoint);
         messaging.send(message, endpoint);
     }
 
@@ -233,7 +233,7 @@ public class AccordMessageSink implements MessageSink
         Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
         Message<Request> message = Message.out(verb, request);
         InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
-        logger.debug("Sending {} {} to {}", verb, message.payload, endpoint);
+        logger.trace("Sending {} {} to {}", verb, message.payload, endpoint);
         messaging.sendWithCallback(message, endpoint, new AccordCallback<>(executor, (Callback<Reply>) callback, endpointMapper));
     }
 
@@ -246,7 +246,7 @@ public class AccordMessageSink implements MessageSink
             responseMsg = responseMsg.withFlag(MessageFlag.NOT_FINAL);
         checkReplyType(reply, respondTo);
         InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode);
-        logger.debug("Replying {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
+        logger.trace("Replying {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
         messaging.send(responseMsg, endpoint);
     }
 
@@ -256,7 +256,7 @@ public class AccordMessageSink implements MessageSink
         ResponseContext respondTo = (ResponseContext) replyContext;
         Message<?> responseMsg = Message.failureResponse(RequestFailureReason.UNKNOWN, failure, respondTo);
         InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode);
-        logger.debug("Replying with failure {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
+        logger.trace("Replying with failure {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
         messaging.send(responseMsg, endpoint);
     }
 

src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java

@@ -18,12 +18,9 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.function.BiFunction;
-
 import javax.annotation.Nullable;
 
 import accord.api.Agent;
@@ -33,12 +30,13 @@ import accord.api.ProgressLog;
 import accord.impl.AbstractSafeCommandStore;
 import accord.local.CommandsForKey;
 import accord.impl.CommandsSummary;
-import accord.local.Command;
 import accord.local.CommandStores.RangesForEpoch;
 import accord.local.NodeTimeService;
 import accord.local.PreLoadContext;
 import accord.primitives.AbstractKeys;
+import accord.primitives.AbstractRanges;
 import accord.primitives.Deps;
+import accord.primitives.Range;
 import accord.primitives.Ranges;
 import accord.primitives.Routables;
 import accord.primitives.Seekables;
@@ -46,27 +44,27 @@ import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
 
-import static accord.primitives.Routable.Domain.Range;
-
 public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey>
 {
     private final Map<TxnId, AccordSafeCommand> commands;
     private final NavigableMap<Key, AccordSafeCommandsForKey> commandsForKeys;
     private final NavigableMap<Key, AccordSafeTimestampsForKey> timestampsForKeys;
+    private final @Nullable AccordSafeCommandsForRanges commandsForRanges;
     private final AccordCommandStore commandStore;
     private final RangesForEpoch ranges;
-    CommandsForRanges.Updater rangeUpdates = null;
 
     public AccordSafeCommandStore(PreLoadContext context,
                                   Map<TxnId, AccordSafeCommand> commands,
                                   NavigableMap<Key, AccordSafeTimestampsForKey> timestampsForKey,
                                   NavigableMap<Key, AccordSafeCommandsForKey> commandsForKey,
+                                  @Nullable AccordSafeCommandsForRanges commandsForRanges,
                                   AccordCommandStore commandStore)
     {
         super(context);
         this.commands = commands;
         this.timestampsForKeys = timestampsForKey;
         this.commandsForKeys = commandsForKey;
+        this.commandsForRanges = commandsForRanges;
         this.commandStore = commandStore;
         this.ranges = commandStore.updateRangesForEpoch();
     }
@@ -185,27 +183,57 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
                 get(key).registerHistorical(this, txnId);
             });
         });
-        CommandsForRanges commandsForRanges = commandStore.commandsForRanges();
-        deps.rangeDeps.forEachUniqueTxnId(allRanges, txnId -> {
-            if (commandsForRanges.containsLocally(txnId))
-                return;
+        for (int i = 0; i < deps.rangeDeps.rangeCount(); i++)
+        {
+            Range range = deps.rangeDeps.range(i);
+            if (!allRanges.intersects(range))
+                continue;
+            deps.rangeDeps.forEach(range, txnId -> {
+                // TODO (desired, efficiency): this can be made more efficient by batching by epoch
+                if (ranges.coordinates(txnId).intersects(range))
+                    return; // already coordinates, no need to replicate
+                if (!ranges.allBefore(txnId.epoch()).intersects(range))
+                    return;
 
-            Ranges ranges = deps.rangeDeps.ranges(txnId);
-            if (this.ranges.coordinates(txnId).intersects(ranges))
-                return; // already coordinates, no need to replicate
-            if (!this.ranges.allBefore(txnId.epoch()).intersects(ranges))
-                return;
-
-            updateRanges().mergeRemote(txnId, ranges.slice(allRanges), Ranges::with);
-        });
+                commandStore.diskCommandsForRanges().mergeHistoricalTransaction(txnId, Ranges.single(range).slice(allRanges), Ranges::with);
+            });
+        }
     }
 
     private <O> O mapReduce(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
     {
-        accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate);
+        accumulate = mapReduceForRange(keysOrRanges, slice, map, accumulate);
         return mapReduceForKey(keysOrRanges, slice, map, accumulate);
     }
 
+    private <O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
+    {
+        if (commandsForRanges == null)
+            return accumulate;
+        switch (keysOrRanges.domain())
+        {
+            case Key:
+            {
+                AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges.slice(slice, Routables.Slice.Minimal);
+                if (!commandsForRanges.ranges().intersects(keys))
+                    return accumulate;
+                accumulate = map.apply(commandsForRanges.current(), accumulate);
+            }
+            break;
+            case Range:
+            {
+                AbstractRanges ranges = (AbstractRanges) keysOrRanges.slice(slice, Routables.Slice.Minimal);
+                if (!commandsForRanges.ranges().intersects(ranges))
+                    return accumulate;
+                accumulate = map.apply(commandsForRanges.current(), accumulate);
+            }
+            break;
+            default:
+                throw new AssertionError("Unknown domain: " + keysOrRanges.domain());
+        }
+        return accumulate;
+    }
+
     private <O> O mapReduceForKey(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandsSummary, O, O> map, O accumulate)
     {
         switch (keysOrRanges.domain())
@@ -260,33 +288,6 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
         }, accumulate);
     }
 
-    @Override
-    protected void update(Command prev, Command updated)
-    {
-        super.update(prev, updated);
-
-        if (updated.txnId().domain() == Range && CommandsForKey.needsUpdate(prev, updated))
-        {
-            Seekables<?, ?> keysOrRanges = updated.keysOrRanges();
-            if (keysOrRanges == null) keysOrRanges = prev.keysOrRanges();
-            if (keysOrRanges == null)
-                return;
-
-            List<TxnId> waitingOn;
-            // TODO (required): this is faulty: we cannot simply save the raw transaction ids, as they may be for other ranges
-            if (updated.partialDeps() == null) waitingOn = Collections.emptyList();
-            else waitingOn = updated.partialDeps().txnIds();
-            updateRanges().put(updated.txnId(), (Ranges)keysOrRanges, updated.saveStatus(), updated.executeAt(), waitingOn);
-        }
-    }
-
-    protected CommandsForRanges.Updater updateRanges()
-    {
-        if (rangeUpdates == null)
-            rangeUpdates = commandStore.updateRanges();
-        return rangeUpdates;
-    }
-
     @Override
     protected void invalidateSafeState()
     {
@@ -297,14 +298,14 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
 
     public void postExecute(Map<TxnId, AccordSafeCommand> commands,
                             Map<Key, AccordSafeTimestampsForKey> timestampsForKey,
-                            Map<Key, AccordSafeCommandsForKey> commandsForKeys
-                            )
+                            Map<Key, AccordSafeCommandsForKey> commandsForKeys,
+                            @Nullable AccordSafeCommandsForRanges commandsForRanges)
     {
         postExecute();
         commands.values().forEach(AccordSafeState::postExecute);
         timestampsForKey.values().forEach(AccordSafeState::postExecute);
         commandsForKeys.values().forEach(AccordSafeState::postExecute);
-        if (rangeUpdates != null)
-            rangeUpdates.apply();
+        if (commandsForRanges != null)
+            commandsForRanges.postExecute();
     }
 }

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForRanges.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.TxnId;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.utils.Pair;
+
+public class AccordSafeCommandsForRanges implements AccordSafeState<Range, CommandsForRanges>
+{
+    private final AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain;
+    private final Ranges ranges;
+    private boolean invalidated;
+    private CommandsForRanges original, current;
+
+    public AccordSafeCommandsForRanges(Ranges ranges, AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain)
+    {
+        this.ranges = ranges;
+        this.chain = chain;
+    }
+
+    public Ranges ranges()
+    {
+        return ranges;
+    }
+
+    @Override
+    public CommandsForRanges current()
+    {
+        checkNotInvalidated();
+        return current;
+    }
+
+    @Override
+    public void invalidate()
+    {
+        invalidated = true;
+    }
+
+    @Override
+    public boolean invalidated()
+    {
+        return invalidated;
+    }
+
+    @Override
+    public void set(CommandsForRanges update)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public CommandsForRanges original()
+    {
+        checkNotInvalidated();
+        return original;
+    }
+
+    @Override
+    public void preExecute()
+    {
+        checkNotInvalidated();
+        Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>> pair = AsyncChains.getUnchecked(chain);
+        pair.left.close();
+        pair.left.get().entrySet().forEach(e -> pair.right.put(e.getKey(), e.getValue()));
+        current = original = new CommandsForRanges(ranges, pair.right);
+    }
+
+    @Override
+    public void postExecute()
+    {
+        checkNotInvalidated();
+    }
+
+    @Override
+    public AccordCachingState<Range, CommandsForRanges> global()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AccordSafeCommandsForRanges that = (AccordSafeCommandsForRanges) o;
+        return Objects.equals(original, that.original) && Objects.equals(current, that.current);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(original, current);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "AccordSafeCommandsForRange{" +
+               "chain=" + chain +
+               ", invalidated=" + invalidated +
+               ", original=" + original +
+               ", current=" + current +
+               '}';
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -146,6 +146,12 @@ public class AccordService implements IAccordService, Shutdownable
             throw new UnsupportedOperationException("No accord barriers should be executed when accord.enabled = false in cassandra.yaml");
         }
 
+        @Override
+        public long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
+        {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public @Nonnull TxnResult coordinate(@Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime)
         {
@@ -213,6 +219,12 @@ public class AccordService implements IAccordService, Shutdownable
         instance = null;
     }
 
+    @VisibleForTesting
+    public static void unsafeSetNoop()
+    {
+        instance = NOOP_SERVICE;
+    }
+
     public static boolean isSetup()
     {
         return instance != null;

src/java/org/apache/cassandra/service/accord/AccordStateCache.java

@@ -17,8 +17,11 @@
  */
 package org.apache.cassandra.service.accord;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -68,9 +71,9 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
 
     static class Stats
     {
-        private long queries;
-        private long hits;
-        private long misses;
+        long queries;
+        long hits;
+        long misses;
     }
 
     private ImmutableList<Instance<?, ?, ?>> instances = ImmutableList.of();
@@ -83,6 +86,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
 
     @VisibleForTesting
     final AccordStateCacheMetrics metrics;
+    final Stats stats = new Stats();
 
     public AccordStateCache(ExecutorPlus loadExecutor, ExecutorPlus saveExecutor, long maxSizeInBytes, AccordStateCacheMetrics metrics)
     {
@@ -141,34 +145,41 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         while (iter.hasNext() && bytesCached > maxSizeInBytes)
         {
             AccordCachingState<?, ?> node = iter.next();
-            checkState(node.references == 0);
+            maybeEvict(node);
+        }
+    }
 
-            /*
-             * TODO (expected, efficiency):
-             *    can this be reworked so we're not skipping unevictable nodes everytime we try to evict?
-             */
-            Status status = node.status(); // status() call completes (if completeable)
-            switch (status)
-            {
-                default: throw new IllegalStateException("Unhandled status " + status);
-                case LOADED:
-                    unlink(node);
-                    evict(node);
-                    break;
-                case MODIFIED:
-                    // schedule a save to disk, keep linked and in the cache map
-                    Instance<?, ?, ?> instance = instanceForNode(node);
-                    node.save(saveExecutor, instance.saveFunction);
-                    maybeUpdateSize(node, instance.heapEstimator);
-                    break;
-                case SAVING:
-                    // skip over until completes to LOADED or FAILED_TO_SAVE
-                    break;
-                case FAILED_TO_SAVE:
-                    // TODO (consider): panic when a save fails
-                    // permanently unlink, but keep in the map
-                    unlink(node);
-            }
+    @VisibleForTesting
+    public boolean maybeEvict(AccordCachingState<?, ?> node)
+    {
+        checkState(node.references == 0);
+
+        /*
+         * TODO (expected, efficiency):
+         *    can this be reworked so we're not skipping unevictable nodes everytime we try to evict?
+         */
+        Status status = node.status(); // status() call completes (if completeable)
+        switch (status)
+        {
+            default: throw new IllegalStateException("Unhandled status " + status);
+            case LOADED:
+                unlink(node);
+                evict(node);
+                return true;
+            case MODIFIED:
+                // schedule a save to disk, keep linked and in the cache map
+                Instance<?, ?, ?> instance = instanceForNode(node);
+                node.save(saveExecutor, instance.saveFunction);
+                maybeUpdateSize(node, instance.heapEstimator);
+                return false;
+            case SAVING:
+                // skip over until completes to LOADED or FAILED_TO_SAVE
+                return false;
+            case FAILED_TO_SAVE:
+                // TODO (consider): panic when a save fails
+                // permanently unlink, but keep in the map
+                unlink(node);
+                return false;
         }
     }
 
@@ -189,12 +200,14 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         instance.bytesCached -= node.lastQueriedEstimatedSizeOnHeap;
 
         if (node.status() == LOADED && VALIDATE_LOAD_ON_EVICT)
-            instanceForNode(node).validateLoadEvicted(node);
+            instance.validateLoadEvicted(node);
 
         if (!node.hasListeners())
         {
             AccordCachingState<?, ?> self = instances.get(node.index).cache.remove(node.key());
             checkState(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self);
+            if (instance.listeners != null)
+                instance.listeners.forEach(l -> l.onEvict((AccordCachingState) node));
         }
         else
         {
@@ -240,7 +253,19 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         return instance(keyClass, valClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, AccordCachingState.defaultFactory());
     }
 
-    public class Instance<K, V, S extends AccordSafeState<K, V>> implements CacheSize
+    public Collection<Instance<?, ? ,? >> instances()
+    {
+        return instances;
+    }
+
+    public interface Listener<K, V>
+    {
+        default void onAdd(AccordCachingState<K, V> state) {}
+        default void onRelease(AccordCachingState<K, V> state) {}
+        default void onEvict(AccordCachingState<K, V> state) {}
+    }
+
+    public class Instance<K, V, S extends AccordSafeState<K, V>> implements CacheSize, Iterable<AccordCachingState<K, V>>
     {
         private final int index;
         private final Class<K> keyClass;
@@ -257,6 +282,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         private final Stats stats = new Stats();
         private final Map<Object, AccordCachingState<?, ?>> cache = new HashMap<>();
         private final AccordCachingState.Factory<K, V> nodeFactory;
+        private List<Listener<K, V>> listeners = null;
 
         public Instance(
             int index, Class<K> keyClass,
@@ -278,13 +304,36 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
             this.nodeFactory = nodeFactory;
         }
 
+        public void register(Listener<K, V> l)
+        {
+            if (listeners == null)
+                listeners = new ArrayList<>();
+            listeners.add(l);
+        }
+
+        public void unregister(Listener<K, V> l)
+        {
+            if (listeners == null)
+                throw new AssertionError("No listeners exist");
+            if (!listeners.remove(l))
+                throw new AssertionError("Listener was not registered");
+            if (listeners.isEmpty())
+                listeners = null;
+        }
+
         public Stream<AccordCachingState<K, V>> stream()
         {
             return cache.entrySet().stream()
-                        .filter(e -> keyClass.isAssignableFrom(e.getKey().getClass()))
+                        .filter(e -> instanceForNode(e.getValue()) == this)
                         .map(e -> (AccordCachingState<K, V>) e.getValue());
         }
 
+        @Override
+        public Iterator<AccordCachingState<K, V>> iterator()
+        {
+            return stream().iterator();
+        }
+
         public S acquireOrInitialize(K key, Function<K, V> valueFactory)
         {
             incrementCacheQueries();
@@ -295,6 +344,11 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
                 node = nodeFactory.create(key, index);
                 node.initialize(valueFactory.apply(key));
                 cache.put(key, node);
+                if (listeners != null)
+                {
+                    AccordCachingState<K, V> finalNode = node;
+                    listeners.forEach(l -> l.onAdd(finalNode));
+                }
             }
             AccordCachingState<K, V> acquired = acquireExisting(node, true);
             Invariants.checkState(acquired != null, "%s could not be acquired", node);
@@ -350,6 +404,8 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
             node.references++;
 
             cache.put(key, node);
+            if (listeners != null)
+                listeners.forEach(l -> l.onAdd(node));
             maybeUpdateSize(node, heapEstimator);
             metrics.objectSize.update(node.lastQueriedEstimatedSizeOnHeap);
             maybeEvictSomeNodes();
@@ -402,6 +458,9 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
 
             maybeUpdateSize(node, heapEstimator);
 
+            if (listeners != null)
+                listeners.forEach(l -> l.onRelease(node));
+
             if (--node.references == 0)
             {
                 Status status = node.status(); // status() completes
@@ -508,18 +567,34 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         {
             instanceMetrics.requests.mark();
             metrics.requests.mark();
+            stats.queries++;
+            AccordStateCache.this.stats.queries++;
         }
 
         private void incrementCacheHits()
         {
             instanceMetrics.hits.mark();
             metrics.hits.mark();
+            stats.hits++;
+            AccordStateCache.this.stats.hits++;
         }
 
         private void incrementCacheMisses()
         {
             instanceMetrics.misses.mark();
             metrics.misses.mark();
+            stats.misses++;
+            AccordStateCache.this.stats.misses++;
+        }
+
+        public Stats stats()
+        {
+            return stats;
+        }
+
+        public Stats globalStats()
+        {
+            return AccordStateCache.this.stats;
         }
 
         @VisibleForTesting
@@ -557,6 +632,30 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
         {
             return bytesCached;
         }
+
+        public long globalAllocated()
+        {
+            return AccordStateCache.this.bytesCached;
+        }
+
+        public int globalReferencedEntries()
+        {
+            return AccordStateCache.this.numReferencedEntries();
+        }
+
+        public int globalUnreferencedEntries()
+        {
+            return AccordStateCache.this.numUnreferencedEntries();
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Instance{" +
+                   "index=" + index +
+                   ", keyClass=" + keyClass +
+                   '}';
+        }
     }
 
     @VisibleForTesting

src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java

@@ -47,7 +47,7 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
     {
         // TODO (desired): need a non-blocking way to inform CMS of an unknown epoch and add callback to it's receipt
 //        ClusterMetadataService.instance().maybeCatchup(message.epoch());
-        logger.debug("Receiving {} from {}", message.payload, message.from());
+        logger.trace("Receiving {} from {}", message.payload, message.from());
         T request = message.payload;
 
         if (request.type().hasSideEffects())

src/java/org/apache/cassandra/service/accord/CommandsForRanges.java

@@ -19,49 +19,25 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.NavigableMap;
 import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.function.BiFunction;
-import java.util.function.Function;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-
-import accord.api.Key;
-import accord.api.RoutingKey;
-import accord.local.CommandsForKey;
 import accord.impl.CommandsSummary;
-import accord.local.Command;
+import accord.local.SafeCommandStore.CommandFunction;
+import accord.local.SafeCommandStore.TestDep;
+import accord.local.SafeCommandStore.TestStartedAt;
+import accord.local.SafeCommandStore.TestStatus;
 import accord.local.SaveStatus;
-import accord.primitives.AbstractKeys;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
-import accord.primitives.RoutableKey;
-import accord.primitives.Seekable;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
-import accord.utils.Invariants;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.utils.Interval;
-import org.apache.cassandra.utils.IntervalTree;
 
-import static accord.local.SafeCommandStore.*;
 import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
 import static accord.local.SafeCommandStore.TestDep.WITH;
 import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
@@ -69,509 +45,119 @@ import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
 import static accord.local.Status.Stable;
 import static accord.local.Status.Truncated;
 
-public class CommandsForRanges
+public class CommandsForRanges implements CommandsSummary
 {
-    public enum TxnType
+    private final Ranges ranges;
+    private final NavigableMap<Timestamp, CommandsForRangesLoader.Summary> map;
+
+    public CommandsForRanges(Ranges ranges, NavigableMap<TxnId, CommandsForRangesLoader.Summary> map)
     {
-        UNKNOWN, LOCAL, REMOTE;
-
-        private boolean isSafeToMix(TxnType other)
-        {
-            if (this == UNKNOWN || other == UNKNOWN) return true;
-            return this == other;
-        }
-    }
-
-    public static final class RangeCommandSummary implements Comparable<RangeCommandSummary>
-    {
-        public final TxnId txnId;
-        public final Ranges ranges;
-        public final SaveStatus status;
-        public final @Nullable Timestamp executeAt;
-        public final List<TxnId> deps;
-
-        RangeCommandSummary(TxnId txnId, Ranges ranges, SaveStatus status, @Nullable Timestamp executeAt, List<TxnId> deps)
-        {
-            this.txnId = txnId;
-            this.ranges = ranges;
-            this.status = status;
-            this.executeAt = executeAt;
-            this.deps = deps;
-        }
-
-        public boolean equalsDeep(RangeCommandSummary other)
-        {
-            return Objects.equals(txnId, other.txnId)
-                   && Objects.equals(ranges, other.ranges)
-                   && status == other.status
-                   && Objects.equals(executeAt, other.executeAt)
-                   && Objects.equals(deps, other.deps);
-        }
-
-        @Override
-        public boolean equals(Object o)
-        {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            RangeCommandSummary that = (RangeCommandSummary) o;
-            return txnId.equals(that.txnId);
-        }
-
-        @Override
-        public int hashCode()
-        {
-            return Objects.hash(txnId);
-        }
-
-        @Override
-        public String toString()
-        {
-            return "RangeCommandSummary{" +
-                   "txnId=" + txnId +
-                   ", status=" + status +
-                   ", ranges=" + ranges +
-                   '}';
-        }
-
-        public RangeCommandSummary withRanges(Ranges ranges, BiFunction<? super Ranges, ? super Ranges, ? extends Ranges> remappingFunction)
-        {
-            return new RangeCommandSummary(txnId, remappingFunction.apply(this.ranges, ranges), status, executeAt, deps);
-        }
-
-        @Override
-        public int compareTo(RangeCommandSummary other)
-        {
-            // Used in IntervalTree with the expecation that compareTo uniquely identifies an RangeCommandSummary
-            return txnId.compareTo(other.txnId);
-        }
-    }
-
-    public static abstract class AbstractBuilder<T extends AbstractBuilder<T>>
-    {
-        protected final Set<TxnId> localTxns = new HashSet<>();
-        protected final TreeMap<TxnId, RangeCommandSummary> txnToRange = new TreeMap<>();
-        protected final IntervalTree.Builder<RoutableKey, RangeCommandSummary, Interval<RoutableKey, RangeCommandSummary>> rangeToTxn = new IntervalTree.Builder<>();
-
-        public TxnType type(TxnId txnId)
-        {
-            if (!txnToRange.containsKey(txnId)) return TxnType.UNKNOWN;
-            return localTxns.contains(txnId) ? TxnType.LOCAL : TxnType.REMOTE;
-        }
-
-        public T put(TxnId txnId, Ranges ranges, SaveStatus status, Timestamp execteAt, List<TxnId> dependsOn)
-        {
-            remove(txnId);
-            put(new RangeCommandSummary(txnId, ranges, status, execteAt, dependsOn));
-            //noinspection unchecked
-            return (T) this;
-        }
-
-        private void put(RangeCommandSummary summary)
-        {
-            TxnId txnId = summary.txnId;
-            localTxns.add(txnId);
-            txnToRange.put(txnId, summary);
-            addRanges(summary);
-        }
-
-        private void addRanges(RangeCommandSummary summary)
-        {
-            for (Range range : summary.ranges)
-                rangeToTxn.add(Interval.create(normalize(range.start(), range.startInclusive(), true),
-                                               normalize(range.end(), range.endInclusive(), false),
-                                               summary));
-        }
-
-        public T putAll(CommandsForRanges other)
-        {
-            for (TxnId id : other.localCommands)
-            {
-                TxnType thisType = type(id);
-                TxnType otherType = other.type(id);
-                Invariants.checkArgument(thisType.isSafeToMix(otherType), "Attempted to add %s; expected %s but was %s", id, thisType, otherType);
-            }
-            localTxns.addAll(other.localCommands);
-            txnToRange.putAll(other.commandsToRanges);
-            // If "put" was called before for a txn present in "other", to respect the "put" semantics that update must
-            // be removed from "rangeToTxn" (as it got removed from "txnToRange").
-            // The expected common case is that this method is called on an empty builder, so the removeIf is off an
-            // empty list (aka no-op)
-            rangeToTxn.removeIf(data -> other.commandsToRanges.containsKey(data.txnId));
-            rangeToTxn.addAll(other.rangesToCommands);
-            //noinspection unchecked
-            return (T) this;
-        }
-
-        public T mergeRemote(TxnId txnId, Ranges ranges, BiFunction<? super Ranges, ? super Ranges, ? extends Ranges> remappingFunction)
-        {
-            // TODO (durability) : remote ranges are not made durable for now.  If this command is stored in commands table,
-            // then we have a NotWitnessed command with Ranges, which is not expected in accord.local.Command.NotWitnessed.
-            // To properly handle this, the long term storage looks like it will need to store these as well.
-            Invariants.checkArgument(!localTxns.contains(txnId), "Attempted to merge remote txn %s, but this is a local txn", txnId);
-            // accord.impl.CommandTimeseries.mapReduce does the check on status and deps type, and NotWitnessed should match the semantics hard coded in InMemorySafeStore...
-            // in that store, the remote history is only ever included when minStauts == null and deps == ANY... but mapReduce sees accord.local.Status.KnownDeps.hasProposedOrDecidedDeps == false
-            // as a mis-match, so will be excluded... since NotWitnessed will return false it will only be included IFF deps = ANY.
-            // When it comes to the minStatus check, the current usage is "null", "Committed", "Accepted"... so NotWitnessed will only be included in the null case;
-            // the only subtle difference is if minStatus = NotWitnessed, this API will include these but InMemoryStore won't
-            RangeCommandSummary oldValue = txnToRange.get(txnId);
-            RangeCommandSummary newValue = oldValue == null ?
-                                           new RangeCommandSummary(txnId, ranges, SaveStatus.NotDefined, null, Collections.emptyList())
-                                           : oldValue.withRanges(ranges, remappingFunction);
-            if (oldValue == null || !oldValue.equalsDeep(newValue))
-            {
-                // changes detected... have to update range index
-                rangeToTxn.removeIf(data -> data.txnId.equals(txnId));
-                addRanges(newValue);
-            }
-            //noinspection unchecked
-            return (T) this;
-        }
-
-        public T remove(TxnId txnId)
-        {
-            if (txnToRange.containsKey(txnId))
-            {
-                localTxns.remove(txnId);
-                txnToRange.remove(txnId);
-                rangeToTxn.removeIf(data -> data.txnId.equals(txnId));
-            }
-            //noinspection unchecked
-            return (T) this;
-        }
-
-        public T map(Function<? super RangeCommandSummary, ? extends RangeCommandSummary> mapper)
-        {
-            for (TxnId id : new TreeSet<>(txnToRange.keySet()))
-            {
-                RangeCommandSummary summary = txnToRange.get(id);
-                RangeCommandSummary update = mapper.apply(summary);
-                if (summary.equals(update))
-                    continue;
-                remove(summary.txnId);
-                if (update != null)
-                    put(update);
-            }
-            //noinspection unchecked
-            return (T) this;
-        }
-    }
-
-    public static class Builder extends AbstractBuilder<Builder>
-    {
-        public CommandsForRanges build()
-        {
-            CommandsForRanges cfr = new CommandsForRanges();
-            cfr.set(this);
-            return cfr;
-        }
-    }
-
-    public class Updater extends AbstractBuilder<Updater>
-    {
-        private Updater()
-        {
-            putAll(CommandsForRanges.this);
-        }
-
-        public void apply()
-        {
-            CommandsForRanges.this.set(this);
-        }
-    }
-
-    private ImmutableSet<TxnId> localCommands;
-    private ImmutableSortedMap<TxnId, RangeCommandSummary> commandsToRanges;
-    private IntervalTree<RoutableKey, RangeCommandSummary, Interval<RoutableKey, RangeCommandSummary>> rangesToCommands;
-    @Nullable
-    private Timestamp maxRedundant;
-
-    public CommandsForRanges()
-    {
-        localCommands = ImmutableSet.of();
-        commandsToRanges = ImmutableSortedMap.of();
-        rangesToCommands = IntervalTree.emptyTree();
-    }
-
-    private void set(AbstractBuilder<?> builder)
-    {
-        this.localCommands = ImmutableSet.copyOf(builder.localTxns);
-        this.commandsToRanges = ImmutableSortedMap.copyOf(builder.txnToRange);
-        this.rangesToCommands = builder.rangeToTxn.build();
-    }
-
-    public TxnType type(TxnId txnId)
-    {
-        if (!commandsToRanges.containsKey(txnId)) return TxnType.UNKNOWN;
-        return localCommands.contains(txnId) ? TxnType.LOCAL : TxnType.REMOTE;
-    }
-
-    @VisibleForTesting
-    Set<TxnId> knownIds()
-    {
-        return commandsToRanges.keySet();
-    }
-
-    @VisibleForTesting
-    IntervalTree<RoutableKey, RangeCommandSummary, Interval<RoutableKey, RangeCommandSummary>> tree()
-    {
-        return rangesToCommands;
-    }
-
-    public @Nullable Timestamp maxRedundant()
-    {
-        return maxRedundant;
-    }
-
-    public static boolean needsUpdate(Command prev, Command updated)
-    {
-        return CommandsForKey.needsUpdate(prev, updated);
-    }
-
-    public boolean containsLocally(TxnId txnId)
-    {
-        return localCommands.contains(txnId);
-    }
-
-    public Iterable<CommandsSummary> search(AbstractKeys<Key> keys)
-    {
-        // group by the table, as ranges are based off TokenKey, which is scoped to a range
-        Map<TableId, List<Key>> groupByTable = new TreeMap<>();
-        for (Key key : keys)
-            groupByTable.computeIfAbsent(((PartitionKey) key).table(), ignore -> new ArrayList<>()).add(key);
-        return () -> new AbstractIterator<CommandsSummary>()
-        {
-            Iterator<TableId> tblIt = groupByTable.keySet().iterator();
-            Iterator<Map.Entry<Range, Set<RangeCommandSummary>>> rangeIt;
-
-            @Override
-            protected CommandsSummary computeNext()
-            {
-                while (true)
-                {
-                    if (rangeIt != null && rangeIt.hasNext())
-                    {
-                        Map.Entry<Range, Set<RangeCommandSummary>> next = rangeIt.next();
-                        return result(next.getKey(), next.getValue());
-                    }
-                    rangeIt = null;
-                    if (!tblIt.hasNext())
-                    {
-                        tblIt = null;
-                        return endOfData();
-                    }
-                    TableId tbl = tblIt.next();
-                    List<Key> keys = groupByTable.get(tbl);
-                    Map<Range, Set<RangeCommandSummary>> groupByRange = new TreeMap<>(Range::compare);
-                    for (Key key : keys)
-                    {
-                        List<Interval<RoutableKey, RangeCommandSummary>> matches = rangesToCommands.matches(key);
-                        if (matches.isEmpty())
-                            continue;
-                        for (Interval<RoutableKey, RangeCommandSummary> interval : matches)
-                            groupByRange.computeIfAbsent(toRange(interval), ignore -> new HashSet<>()).add(interval.data);
-                    }
-                    rangeIt = groupByRange.entrySet().iterator();
-                }
-            }
-        };
-    }
-
-    private static Range toRange(Interval<RoutableKey, RangeCommandSummary> interval)
-    {
-        AccordRoutingKey start = (AccordRoutingKey) interval.min;
-        if (!(start instanceof AccordRoutingKey.SentinelKey))
-            start = new TokenKey(start.table(), start.token().decreaseSlightly());
-        AccordRoutingKey end = (AccordRoutingKey) interval.max;
-        // TODO (required, correctness) : accord doesn't support wrap around, so decreaseSlightly may fail in some cases
-        // TODO (required, correctness) : this logic is mostly used for testing, so is it actually safe for all partitioners?
-        return new TokenRange(start, end);
-    }
-
-    @Nullable
-    public CommandsSummary search(Range range)
-    {
-        List<RangeCommandSummary> matches = rangesToCommands.search(Interval.create(normalize(range.start(), range.startInclusive(), true),
-                                                                                    normalize(range.end(), range.endInclusive(), false)));
-        return result(range, matches);
-    }
-
-    private CommandsSummary result(Seekable seekable, Collection<RangeCommandSummary> matches)
-    {
-        if (matches.isEmpty())
-            return null;
-        return new Holder(seekable, matches);
-    }
-
-    public int size()
-    {
-        return rangesToCommands.intervalCount();
-    }
-
-    public Updater update()
-    {
-        return new Updater();
+        this.ranges = ranges;
+        this.map = (NavigableMap<Timestamp, CommandsForRangesLoader.Summary>) (NavigableMap<?, ?>) map;
     }
 
     @Override
-    public String toString()
+    public <P1, T> T mapReduceFull(TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
     {
-        return rangesToCommands.unbuild().toString();
+        return mapReduce(testTxnId, testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, accumulate);
     }
 
-    private static RoutingKey normalize(RoutingKey key, boolean inclusive, boolean upOrDown)
+    @Override
+    public <P1, T> T mapReduceActive(Timestamp startedBefore, Txn.Kind.Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
     {
-        while (true)
+        return mapReduce(startedBefore, null, testKind, STARTED_BEFORE, ANY_DEPS, ANY_STATUS, map, p1, accumulate);
+    }
+
+    private <P1, T> T mapReduce(@Nonnull Timestamp testTimestamp, @Nullable TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
+    {
+        // TODO (required): reconsider how we build this, to avoid having to provide range keys in order (or ensure our range search does this for us)
+        Map<Range, List<CommandsForRangesLoader.Summary>> collect = new TreeMap<>(Range::compare);
+        NavigableMap<Timestamp, CommandsForRangesLoader.Summary> submap;
+        switch (testStartedAt)
         {
-            if (inclusive) return key;
-            AccordRoutingKey ak = (AccordRoutingKey) key;
-            switch (ak.kindOfRoutingKey())
+            case STARTED_AFTER:
+                submap = this.map.tailMap(testTimestamp, false);
+                break;
+            case STARTED_BEFORE:
+                submap = this.map.headMap(testTimestamp, false);
+                break;
+            case ANY:
+                submap = this.map;
+                break;
+            default:
+                throw new AssertionError("Unknown started at: " + testStartedAt);
+        }
+        submap.values().forEach((summary -> {
+            if (!testKind.test(summary.txnId.kind()))
+                return;
+
+            // range specific logic... ranges don't update CommandsForRange based off the life cycle and instead
+            // merge the cache with the disk state; so exclude states that should get removed from CommandsFor*
+            if (summary.saveStatus.compareTo(SaveStatus.Erased) >= 0)
+                return;
+
+            switch (testStatus)
             {
-                case SENTINEL:
-                    // TODO (required, correctness): this doesn't work
-                    key = ak.asSentinelKey().toTokenKeyBroken();
+                default: throw new AssertionError("Unhandled TestStatus: " + testStatus);
+                case ANY_STATUS:
+                    //TODO (now, symitry): how do we map to TRANSITIVELY_KNOWN?
+                    break;
+                case IS_PROPOSED:
+                    switch (summary.saveStatus.status)
+                    {
+                        default: return;
+                        case PreCommitted:
+                        case Committed:
+                        case Accepted:
+                        case AcceptedInvalidate:
+                    }
+                    break;
+                case IS_STABLE:
+                    if (!summary.saveStatus.hasBeen(Stable) || summary.saveStatus.hasBeen(Truncated))
+                        return;
+            }
+
+            if (testDep != ANY_DEPS)
+            {
+                // ! status.hasInfo
+                //TODO (now, reuse): should this just check if known?
+                if (!(summary.saveStatus.compareTo(SaveStatus.Accepted) >= 0))
+                    return;
+
+                Timestamp executeAt = summary.executeAt;
+                if (executeAt.compareTo(testTxnId) <= 0)
+                    return;
+
+                // TODO (required): we must ensure these txnId are limited to those we intersect in this command store
+                // We are looking for transactions A that have (or have not) B as a dependency.
+                // If B covers ranges [1..3] and A covers [2..3], but the command store only covers ranges [1..2],
+                // we could have A adopt B as a dependency on [3..3] only, and have that A intersects B on this
+                // command store, but also that there is no dependency relation between them on the overlapping
+                // key range [2..2].
+
+                // This can lead to problems on recovery, where we believe a transaction is a dependency
+                // and so it is safe to execute, when in fact it is only a dependency on a different shard
+                // (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
+                // TODO (required): consider this some more
+                if ((testDep == WITH) == !summary.depsIds.contains(testTxnId))
+                    return;
+            }
+
+            // TODO (required): ensure we are excluding any ranges that are now shard-redundant (not sure if this is enforced yet)
+            for (Range range : summary.ranges)
+            {
+                if (!this.ranges.intersects(range))
                     continue;
-                case TOKEN:
-                    TokenKey tk = ak.asTokenKey();
-                    // TODO (required, correctness): this doesn't work for ordered partitioner
-                    return tk.withToken(upOrDown ? tk.token().increaseSlightly() : tk.token().decreaseSlightly());
-                default:
-                    throw new IllegalArgumentException("Unknown kind: " + ak.kindOfRoutingKey());
+                collect.computeIfAbsent(range, ignore -> new ArrayList<>()).add(summary);
             }
+        }));
+
+        for (Map.Entry<Range, List<CommandsForRangesLoader.Summary>> e : collect.entrySet())
+        {
+            for (CommandsForRangesLoader.Summary command : e.getValue())
+                accumulate = map.apply(p1, e.getKey(), command.txnId, command.executeAt, accumulate);
         }
+
+        return accumulate;
     }
-
-    private static class Holder implements CommandsSummary
-    {
-        private final Seekable keyOrRange;
-        private final Collection<RangeCommandSummary> matches;
-
-        private Holder(Seekable keyOrRange, Collection<RangeCommandSummary> matches)
-        {
-            this.keyOrRange = keyOrRange;
-            this.matches = matches;
-        }
-
-        @Override
-        public <P1, T> T mapReduceFull(TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
-        {
-            return mapReduce(testTxnId, testTxnId, testKind, testStartedAt, testDep, testStatus, map, p1, accumulate);
-        }
-
-        @Override
-        public <P1, T> T mapReduceActive(Timestamp startedBefore, Txn.Kind.Kinds testKind, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
-        {
-            return mapReduce(startedBefore, null, testKind, STARTED_BEFORE, ANY_DEPS, ANY_STATUS, map, p1, accumulate);
-        }
-
-        private <P1, T> T mapReduce(@Nonnull Timestamp testTimestamp, @Nullable TxnId testTxnId, Txn.Kind.Kinds testKind, TestStartedAt testStartedAt, TestDep testDep, TestStatus testStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate)
-        {
-            // TODO (required): reconsider how we build this, to avoid having to provide range keys in order (or ensure our range search does this for us)
-            Map<Range, List<RangeCommandSummary>> collect = new TreeMap<>(Range::compare);
-            matches.forEach((summary -> {
-                if (summary.status.compareTo(SaveStatus.Erased) >= 0)
-                    return;
-
-                switch (testStartedAt)
-                {
-                    default: throw new AssertionError();
-                    case STARTED_AFTER:
-                        if (summary.txnId.compareTo(testTimestamp) <= 0) return;
-                        else break;
-                    case STARTED_BEFORE:
-                        if (summary.txnId.compareTo(testTimestamp) >= 0) return;
-                    case ANY:
-                        if (testDep != ANY_DEPS && (summary.executeAt == null || summary.executeAt.compareTo(testTxnId) < 0))
-                            return;
-                }
-
-                switch (testStatus)
-                {
-                    default: throw new AssertionError("Unhandled TestStatus: " + testStatus);
-                    case ANY_STATUS:
-                        break;
-                    case IS_PROPOSED:
-                        switch (summary.status)
-                        {
-                            default: return;
-                            case PreCommitted:
-                            case Committed:
-                            case Accepted:
-                        }
-                        break;
-                    case IS_STABLE:
-                        if (!summary.status.hasBeen(Stable) || summary.status.hasBeen(Truncated))
-                            return;
-                }
-
-                if (!testKind.test(summary.txnId.kind()))
-                    return;
-
-                if (testDep != ANY_DEPS)
-                {
-                    if (!summary.status.known.deps.hasProposedOrDecidedDeps())
-                        return;
-
-                    // TODO (required): we must ensure these txnId are limited to those we intersect in this command store
-                    // We are looking for transactions A that have (or have not) B as a dependency.
-                    // If B covers ranges [1..3] and A covers [2..3], but the command store only covers ranges [1..2],
-                    // we could have A adopt B as a dependency on [3..3] only, and have that A intersects B on this
-                    // command store, but also that there is no dependency relation between them on the overlapping
-                    // key range [2..2].
-
-                    // This can lead to problems on recovery, where we believe a transaction is a dependency
-                    // and so it is safe to execute, when in fact it is only a dependency on a different shard
-                    // (and that other shard, perhaps, does not know that it is a dependency - and so it is not durably known)
-                    // TODO (required): consider this some more
-                    if ((testDep == WITH) == !summary.deps.contains(testTxnId))
-                        return;
-                }
-
-                // TODO (required): ensure we are excluding any ranges that are now shard-redundant (not sure if this is enforced yet)
-                for (Range range : summary.ranges)
-                    collect.computeIfAbsent(range, ignore -> new ArrayList<>()).add(summary);
-            }));
-
-            for (Map.Entry<Range, List<RangeCommandSummary>> e : collect.entrySet())
-            {
-                for (RangeCommandSummary command : e.getValue())
-                {
-                    T initial = accumulate;
-                    accumulate = map.apply(p1, e.getKey(), command.txnId, command.executeAt, initial);
-                }
-            }
-
-            return accumulate;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "Holder{" +
-                   "keyOrRange=" + keyOrRange +
-                   ", matches=" + matches +
-                   '}';
-        }
-    }
-
-    public void prune(TxnId pruneBefore, Ranges pruneRanges)
-    {
-        class MaxErased { Timestamp v; }
-        MaxErased maxErased = new MaxErased();
-        Updater update = update();
-        update.map(summary -> {
-            if (summary.txnId.compareTo(pruneBefore) >= 0)
-                return summary;
-
-            Ranges newRanges = summary.ranges.subtract(pruneRanges);
-            if (newRanges == summary.ranges || newRanges.equals(summary.ranges))
-                return summary;
-
-            maxErased.v = Timestamp.nonNullOrMax(maxErased.v, summary.executeAt);
-            if (newRanges.isEmpty())
-                return null;
-            return new RangeCommandSummary(summary.txnId, newRanges, summary.status, summary.executeAt, summary.deps);
-        }).apply();
-        maxRedundant = Timestamp.nonNullOrMax(maxRedundant, maxErased.v);
-    }
-
 }

src/java/org/apache/cassandra/service/accord/CommandsForRangesLoader.java

@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.BiFunction;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableMap;
+
+import accord.local.Command;
+import accord.local.DurableBefore;
+import accord.local.SaveStatus;
+import accord.local.Status;
+import accord.primitives.PartialDeps;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable;
+import accord.primitives.Seekables;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.index.accord.RoutesSearcher;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.Pair;
+
+public class CommandsForRangesLoader
+{
+    private final RoutesSearcher searcher = new RoutesSearcher();
+    //TODO (now, durability): find solution for this...
+    private final Map<TxnId, Ranges> historicalTransaction = new HashMap<>();
+    private final AccordCommandStore store;
+
+    public CommandsForRangesLoader(AccordCommandStore store)
+    {
+        this.store = store;
+    }
+
+    public AsyncResult<Pair<Watcher, NavigableMap<TxnId, Summary>>> get(Ranges ranges)
+    {
+        Watcher watcher = fromCache(ranges);
+        ImmutableMap<TxnId, Summary> before = ImmutableMap.copyOf(watcher.get());
+        return AsyncChains.ofCallable(Stage.READ.executor(), () -> get(ranges, before))
+                          .map(map -> Pair.create(watcher, map), store)
+               .beginAsResult();
+    }
+
+    private NavigableMap<TxnId, Summary> get(Ranges ranges, Map<TxnId, Summary> cacheHits)
+    {
+        Set<TxnId> matches = new HashSet<>();
+        for (Range range : ranges)
+            matches.addAll(intersects(range));
+        if (matches.isEmpty())
+            return new TreeMap<>();
+        return load(ranges, cacheHits, matches);
+    }
+
+    private Collection<TxnId> intersects(Range range)
+    {
+        assert range instanceof TokenRange : "Require TokenRange but given " + range.getClass();
+        Set<TxnId> intersects = searcher.intersects(store.id(), (TokenRange) range);
+        if (!historicalTransaction.isEmpty())
+        {
+            if (intersects.isEmpty())
+                intersects = new HashSet<>();
+            for (Map.Entry<TxnId, Ranges> e : historicalTransaction.entrySet())
+            {
+                if (e.getValue().intersects(range))
+                    intersects.add(e.getKey());
+            }
+            if (intersects.isEmpty())
+                intersects = Collections.emptySet();
+        }
+        return intersects;
+    }
+
+    public class Watcher implements AccordStateCache.Listener<TxnId, Command>, AutoCloseable
+    {
+        private final Ranges ranges;
+
+        private NavigableMap<TxnId, Summary> summaries = null;
+        private List<AccordCachingState<TxnId, Command>> needToDoubleCheck = null;
+
+        public Watcher(Ranges ranges)
+        {
+            this.ranges = ranges;
+        }
+
+        public NavigableMap<TxnId, Summary> get()
+        {
+            return summaries == null ? Collections.emptyNavigableMap() : summaries;
+        }
+
+        @Override
+        public void onAdd(AccordCachingState<TxnId, Command> n)
+        {
+            if (n.key().domain() != Routable.Domain.Range)
+                return;
+            AccordCachingState.State<TxnId, Command> state = n.state();
+            if (state instanceof AccordCachingState.Loading)
+            {
+                if (needToDoubleCheck == null)
+                    needToDoubleCheck = new ArrayList<>();
+                needToDoubleCheck.add(n);
+                return;
+            }
+            //TODO (now): include FailedToSave?  Most likely need to, but need to improve test coverage to have failed writes
+            if (!(state instanceof AccordCachingState.Loaded
+                  || state instanceof AccordCachingState.Modified
+                  || state instanceof AccordCachingState.Saving))
+                return;
+
+            Command cmd = state.get();
+            if (cmd == null)
+                return;
+            Summary summary = create(cmd, ranges, null);
+            if (summary != null)
+            {
+                if (summaries == null)
+                    summaries = new TreeMap<>();
+                summaries.put(summary.txnId, summary);
+            }
+        }
+
+        @Override
+        public void onEvict(AccordCachingState<TxnId, Command> state)
+        {
+            if (needToDoubleCheck == null)
+                return;
+            if (!needToDoubleCheck.remove(state))
+                return;
+            if (state.state() instanceof AccordCachingState.Loading)
+                return; // can't double check
+            onAdd(state);
+        }
+
+        @Override
+        public void close()
+        {
+            store.commandCache().unregister(this);
+            if (needToDoubleCheck != null)
+            {
+                List<AccordCachingState<TxnId, Command>> copy = needToDoubleCheck;
+                needToDoubleCheck = null;
+                copy.forEach(this::onAdd);
+            }
+            needToDoubleCheck = null;
+        }
+    }
+
+    private Watcher fromCache(Ranges ranges)
+    {
+        Watcher watcher = new Watcher(ranges);
+        store.commandCache().stream().forEach(watcher::onAdd);
+        store.commandCache().register(watcher);
+        return watcher;
+    }
+
+    private NavigableMap<TxnId, Summary> load(Ranges ranges, Map<TxnId, Summary> cacheHits, Collection<TxnId> possibleTxns)
+    {
+        //TODO (now): this logic is kinda duplicate of org.apache.cassandra.service.accord.CommandsForRange.mapReduce
+        // should figure out if this can be improved... also what is correct?
+        DurableBefore durableBefore = store.durableBefore();
+        NavigableMap<TxnId, Summary> map = new TreeMap<>();
+        for (TxnId txnId : possibleTxns)
+        {
+            if (cacheHits.containsKey(txnId))
+                continue;
+            Command cmd = store.loadCommand(txnId);
+            if (cmd == null)
+                continue; // unknown command
+            Summary summary = create(cmd, ranges, durableBefore);
+            if (summary == null)
+                continue;
+            map.put(txnId, summary);
+        }
+        return map;
+    }
+
+    private static Summary create(Command cmd, Ranges cacheRanges, @Nullable DurableBefore durableBefore)
+    {
+        //TODO (now, correctness): C* did Invalidated, accord-core did Erased... what is correct?
+        SaveStatus saveStatus = cmd.saveStatus();
+        if (saveStatus == SaveStatus.Invalidated
+            || saveStatus == SaveStatus.Erased
+            || !saveStatus.hasBeen(Status.PreAccepted))
+            return null;
+        if (cmd.partialTxn() == null)
+            return null;
+
+        Seekables<?, ? extends Seekables<?, ?>> keysOrRanges = cmd.partialTxn().keys();
+        if (keysOrRanges.domain() != Routable.Domain.Range)
+            throw new AssertionError(String.format("Txn keys are not range for %s", cmd.partialTxn()));
+        Ranges ranges = (Ranges) keysOrRanges;
+
+        if (!ranges.intersects(cacheRanges))
+            return null;
+
+        if (durableBefore != null)
+        {
+            Ranges durableAlready = Ranges.of(durableBefore.foldlWithBounds(ranges, (e, accum, start, end) -> {
+                if (e.universalBefore.compareTo(cmd.txnId()) < 0)
+                    return accum;
+                accum.add(new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end));
+                return accum;
+            }, new ArrayList<Range>(), ignore -> false).toArray(Range[]::new));
+            Ranges newRanges = ranges.subtract(durableAlready);
+
+            if (newRanges.isEmpty())
+                return null;
+        }
+
+        PartialDeps partialDeps = cmd.partialDeps();
+        List<TxnId> deps = partialDeps == null ? Collections.emptyList() : partialDeps.txnIds();
+        return new Summary(cmd.txnId(), cmd.executeAt(), saveStatus, ranges, deps);
+    }
+
+    public void mergeHistoricalTransaction(TxnId txnId, Ranges ranges, BiFunction<? super Ranges, ? super Ranges, ? extends Ranges> remappingFunction)
+    {
+        historicalTransaction.merge(txnId, ranges, remappingFunction);
+    }
+
+    public static class Summary
+    {
+        public final TxnId txnId;
+        @Nullable
+        public final Timestamp executeAt;
+        public final SaveStatus saveStatus;
+        public final Ranges ranges;
+        public final List<TxnId> depsIds;
+
+        private Summary(TxnId txnId, @Nullable Timestamp executeAt, SaveStatus saveStatus, Ranges ranges, List<TxnId> depsIds)
+        {
+            this.txnId = txnId;
+            this.executeAt = executeAt;
+            this.saveStatus = saveStatus;
+            this.ranges = ranges;
+            this.depsIds = depsIds;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Summary{" +
+                   "txnId=" + txnId +
+                   ", executeAt=" + executeAt +
+                   ", saveStatus=" + saveStatus +
+                   ", ranges=" + ranges +
+                   ", depsIds=" + depsIds +
+                   '}';
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -57,10 +57,7 @@ public interface IAccordService
 
     IVerbHandler<? extends Request> verbHandler();
 
-    default long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException
-    {
-        throw new UnsupportedOperationException();
-    }
+    long barrierWithRetries(Seekables keysOrRanges, long minEpoch, BarrierType barrierType, boolean isForWrite) throws InterruptedException;
 
     long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite);
 

src/java/org/apache/cassandra/service/accord/IJournal.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import accord.local.SerializerSupport;
+import accord.messages.Message;
+import accord.primitives.TxnId;
+
+public interface IJournal
+{
+    SerializerSupport.MessageProvider makeMessageProvider(TxnId txnId);
+    void appendMessageBlocking(Message message);
+}

src/java/org/apache/cassandra/service/accord/RangeTreeRangeAccessor.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import accord.api.RoutingKey;
+import accord.primitives.Range;
+import org.apache.cassandra.utils.RangeTree;
+
+public enum RangeTreeRangeAccessor implements RangeTree.Accessor<RoutingKey, Range>
+{
+    instance;
+
+    @Override
+    public RoutingKey start(Range range)
+    {
+        return range.start();
+    }
+
+    @Override
+    public RoutingKey end(Range range)
+    {
+        return range.end();
+    }
+
+    @Override
+    public boolean contains(Range range, RoutingKey routingKey)
+    {
+        return range.contains(routingKey);
+    }
+
+    @Override
+    public boolean contains(RoutingKey start, RoutingKey end, RoutingKey routingKey)
+    {
+        if (routingKey.compareTo(start) <= 0)
+            return false;
+        if (routingKey.compareTo(end) > 0)
+            return false;
+        return true;
+    }
+
+    @Override
+    public boolean intersects(Range range, RoutingKey start, RoutingKey end)
+    {
+        if (range.start().compareTo(end) >= 0) return false;
+        if (range.end().compareTo(start) <= 0) return false;
+        return true;
+    }
+
+    @Override
+    public boolean intersects(Range left, Range right)
+    {
+        return left.compareIntersecting(right) == 0;
+    }
+}

src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java

@@ -90,9 +90,9 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
     {
         private static final long EMPTY_SIZE = ObjectSizes.measure(new SentinelKey(null, true));
 
-        private final boolean isMin;
+        public final boolean isMin;
 
-        private SentinelKey(TableId table, boolean isMin)
+        public SentinelKey(TableId table, boolean isMin)
         {
             super(table);
             this.isMin = isMin;

src/java/org/apache/cassandra/service/accord/api/PartitionKey.java

@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import com.google.common.base.Preconditions;
 
 import accord.api.Key;
-import accord.api.RoutingKey;
 import accord.primitives.Routable;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.DecoratedKey;
@@ -96,7 +95,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
     }
 
     @Override
-    public RoutingKey toUnseekable()
+    public TokenKey toUnseekable()
     {
         return new TokenKey(table, token());
     }

src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java

@@ -17,43 +17,30 @@
  */
 package org.apache.cassandra.service.accord.async;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.ImmutableSet;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import accord.api.Key;
-import accord.api.RoutingKey;
+import accord.local.CommandsForKey;
 import accord.local.KeyHistory;
 import accord.local.PreLoadContext;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.Seekables;
-import accord.primitives.TxnId;
+import accord.primitives.*;
 import accord.utils.Invariants;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import accord.utils.async.AsyncResult;
 import accord.utils.async.Observable;
-import org.apache.cassandra.service.accord.AccordCachingState;
-import org.apache.cassandra.service.accord.AccordCommandStore;
-import org.apache.cassandra.service.accord.AccordKeyspace;
-import org.apache.cassandra.service.accord.AccordSafeState;
-import org.apache.cassandra.service.accord.AccordStateCache;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.apache.cassandra.service.accord.*;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.utils.Pair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 
 public class AsyncLoader
 {
@@ -172,15 +159,41 @@ public class AsyncLoader
 
     private AsyncChain<?> referenceAndDispatchReadsForRange(AsyncOperation.Context context)
     {
-        AsyncChain<Set<? extends Key>> overlappingKeys = findOverlappingKeys((Ranges) keysOrRanges);
+        Ranges ranges = (Ranges) keysOrRanges;
 
-        return overlappingKeys.flatMap(keys -> {
-            if (keys.isEmpty())
+        List<AsyncChain<?>> root = new ArrayList<>(ranges.size() + 1);
+        class Watcher implements AccordStateCache.Listener<Key, CommandsForKey>
+        {
+            private final Set<PartitionKey> cached = commandStore.commandsForKeyCache().stream()
+                                                                 .map(n -> (PartitionKey) n.key())
+                                                                 .filter(ranges::contains)
+                                                                 .collect(Collectors.toSet());
+
+            @Override
+            public void onAdd(AccordCachingState<Key, CommandsForKey> state)
+            {
+                PartitionKey pk = (PartitionKey) state.key();
+                if (ranges.contains(pk))
+                    cached.add(pk);
+            }
+        }
+        Watcher watcher = new Watcher();
+        commandStore.commandsForKeyCache().register(watcher);
+        root.add(findOverlappingKeys(ranges).flatMap(keys -> {
+            commandStore.commandsForKeyCache().unregister(watcher);
+            if (keys.isEmpty() && watcher.cached.isEmpty())
                 return AsyncChains.success(null);
+            Set<? extends Key> set = ImmutableSet.<Key>builder().addAll(watcher.cached).addAll(keys).build();
             List<AsyncChain<?>> chains = new ArrayList<>();
-            keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
+            set.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
             return chains.isEmpty() ? AsyncChains.success(null) : AsyncChains.reduce(chains, (a, b) -> null);
-        }, commandStore);
+        }, commandStore));
+
+        AsyncResult<Pair<CommandsForRangesLoader.Watcher, NavigableMap<TxnId, CommandsForRangesLoader.Summary>>> chain = commandStore.diskCommandsForRanges().get(ranges);
+        root.add(chain);
+        context.commandsForRanges = new AccordSafeCommandsForRanges(ranges, chain);
+
+        return AsyncChains.all(root);
     }
 
     private AsyncChain<Set<? extends Key>> findOverlappingKeys(Ranges ranges)
@@ -195,27 +208,14 @@ public class AsyncLoader
 
     private AsyncChain<Set<PartitionKey>> findOverlappingKeys(Range range)
     {
-        Set<PartitionKey> cached = commandStore.commandsForKeyCache().stream()
-                                               .map(n -> (PartitionKey) n.key())
-                                               .filter(range::contains)
-                                               .collect(Collectors.toSet());
         // save to a variable as java gets confused when `.map` is called on the result of asChain
         AsyncChain<Set<PartitionKey>> map = Observable.asChain(callback ->
                                                                AccordKeyspace.findAllKeysBetween(commandStore.id(),
-                                                                                                 toTokenKey(range.start()).token(), range.startInclusive(),
-                                                                                                 toTokenKey(range.end()).token(), range.endInclusive(),
+                                                                                                 (AccordRoutingKey) range.start(), range.startInclusive(),
+                                                                                                 (AccordRoutingKey) range.end(), range.endInclusive(),
                                                                                                  callback),
                                                                Collectors.toSet());
-        return map.map(s -> ImmutableSet.<PartitionKey>builder().addAll(s).addAll(cached).build());
-    }
-
-    private static TokenKey toTokenKey(RoutingKey start)
-    {
-        if (start instanceof TokenKey)
-            return (TokenKey) start;
-        if (start instanceof AccordRoutingKey.SentinelKey)
-            return ((AccordRoutingKey.SentinelKey) start).toTokenKeyBroken();
-        throw new IllegalArgumentException(String.format("Unable to convert RoutingKey %s (type %s) to TokenKey", start, start.getClass()));
+        return map.map(s -> ImmutableSet.<PartitionKey>builder().addAll(s).build());
     }
 
     @VisibleForTesting

src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java

@@ -17,13 +17,11 @@
  */
 package org.apache.cassandra.service.accord.async;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.TreeMap;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
-
 import javax.annotation.Nullable;
 
 import org.slf4j.Logger;
@@ -34,26 +32,26 @@ import accord.api.Key;
 import accord.local.CommandStore;
 import accord.local.PreLoadContext;
 import accord.local.SafeCommandStore;
-import accord.primitives.RoutableKey;
 import accord.primitives.Seekables;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import accord.utils.async.AsyncChains;
 import org.apache.cassandra.service.accord.AccordCommandStore;
 import org.apache.cassandra.service.accord.AccordSafeCommand;
-import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
 import org.apache.cassandra.service.accord.AccordSafeCommandStore;
+import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
+import org.apache.cassandra.service.accord.AccordSafeCommandsForRanges;
 import org.apache.cassandra.service.accord.AccordSafeState;
 import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
 
 import static org.apache.cassandra.service.accord.async.AsyncLoader.txnIds;
+import static org.apache.cassandra.service.accord.async.AsyncOperation.State.COMPLETING;
+import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FAILED;
+import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FINISHED;
 import static org.apache.cassandra.service.accord.async.AsyncOperation.State.INITIALIZED;
 import static org.apache.cassandra.service.accord.async.AsyncOperation.State.LOADING;
 import static org.apache.cassandra.service.accord.async.AsyncOperation.State.PREPARING;
 import static org.apache.cassandra.service.accord.async.AsyncOperation.State.RUNNING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.COMPLETING;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FINISHED;
-import static org.apache.cassandra.service.accord.async.AsyncOperation.State.FAILED;
 
 public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements Runnable, Function<SafeCommandStore, R>
 {
@@ -70,6 +68,8 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
         final HashMap<TxnId, AccordSafeCommand> commands = new HashMap<>();
         final TreeMap<Key, AccordSafeTimestampsForKey> timestampsForKey = new TreeMap<>();
         final TreeMap<Key, AccordSafeCommandsForKey> commandsForKey = new TreeMap<>();
+        @Nullable
+        AccordSafeCommandsForRanges commandsForRanges = null;
 
         void releaseResources(AccordCommandStore commandStore)
         {
@@ -83,6 +83,8 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
             commands.values().forEach(AccordSafeState::revert);
             timestampsForKey.values().forEach(AccordSafeState::revert);
             commandsForKey.values().forEach(AccordSafeState::revert);
+            if (commandsForRanges != null)
+                commandsForRanges.revert();
         }
     }
 
@@ -187,19 +189,9 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
     }
 
     @SuppressWarnings("unchecked")
-    Iterable<RoutableKey> keys()
+    Seekables<?, ?> keys()
     {
-        Seekables<?, ?> keys = preLoadContext.keys();
-        switch (keys.domain())
-        {
-            default:
-                throw new IllegalStateException("Unhandled domain " + keys.domain());
-            case Key:
-                return (Iterable<RoutableKey>) keys;
-            case Range:
-                // TODO (expected): handle ranges
-                return Collections.emptyList();
-        }
+        return preLoadContext.keys();
     }
 
     private void fail(Throwable throwable)
@@ -255,11 +247,11 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
                     return;
                 state(PREPARING);
             case PREPARING:
-                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.commandsForKey);
+                safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.commandsForKey, context.commandsForRanges);
                 state(RUNNING);
             case RUNNING:
                 result = apply(safeStore);
-                safeStore.postExecute(context.commands, context.timestampsForKey, context.commandsForKey);
+                safeStore.postExecute(context.commands, context.timestampsForKey, context.commandsForKey, context.commandsForRanges);
                 context.releaseResources(commandStore);
                 commandStore.completeOperation(safeStore);
                 commandStore.executionOrder().unregister(this);

src/java/org/apache/cassandra/service/accord/async/ExecutionOrder.java

@@ -18,11 +18,20 @@
 package org.apache.cassandra.service.accord.async;
 
 import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.List;
 
-import accord.primitives.RoutableKey;
+import accord.api.Key;
+import accord.api.RoutingKey;
+import accord.primitives.Range;
+import accord.primitives.Seekable;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
 import org.agrona.collections.Object2ObjectHashMap;
+import org.apache.cassandra.service.accord.RangeTreeRangeAccessor;
+import org.apache.cassandra.utils.RTree;
+import org.apache.cassandra.utils.RangeTree;
 
 /**
  * Assists with correct ordering of {@link AsyncOperation} execution wrt each other,
@@ -30,7 +39,85 @@ import org.agrona.collections.Object2ObjectHashMap;
  */
 public class ExecutionOrder
 {
+    private static class Conflicts
+    {
+        private final List<Key> keyConflicts;
+        private final List<Range> rangeConflicts;
+
+        private Conflicts(List<Key> keyConflicts, List<Range> rangeConflicts)
+        {
+            this.keyConflicts = keyConflicts;
+            this.rangeConflicts = rangeConflicts;
+        }
+    }
+    private class RangeState
+    {
+        private final Range range;
+        private final IdentityHashMap<AsyncOperation<?>, Conflicts> operationToConflicts = new IdentityHashMap<>();
+        private Object operationOrQueue;
+
+        public RangeState(Range range, List<Key> keyConflicts, List<Range> rangeConflicts, AsyncOperation<?> operation)
+        {
+            this.range = range;
+            this.operationOrQueue = operation;
+            add(operation, keyConflicts, rangeConflicts);
+        }
+
+        public void add(AsyncOperation<?> operation, List<Key> keyConflicts, List<Range> rangeConflicts)
+        {
+            operationToConflicts.put(operation, new Conflicts(keyConflicts, rangeConflicts));
+        }
+
+        boolean canRun(AsyncOperation<?> operation)
+        {
+            if (operationOrQueue instanceof AsyncOperation<?>)
+            {
+                Invariants.checkState(operationOrQueue == operation);
+                return true;
+            }
+            else
+            {
+                ArrayDeque<AsyncOperation<?>> queue = (ArrayDeque<AsyncOperation<?>>) operationOrQueue;
+                return queue.peek() == operation;
+            }
+        }
+
+        Conflicts remove(AsyncOperation<?> operation)
+        {
+            if (operationOrQueue instanceof AsyncOperation<?>)
+            {
+                Invariants.checkState(operationOrQueue == operation);
+                rangeQueues.remove(range);
+            }
+            else
+            {
+                @SuppressWarnings("unchecked")
+                ArrayDeque<AsyncOperation<?>> queue = (ArrayDeque<AsyncOperation<?>>) operationOrQueue;
+                AsyncOperation<?> head = queue.poll();
+                Invariants.checkState(head == operation);
+
+                if (queue.isEmpty())
+                {
+                    rangeQueues.remove(range);
+                }
+                else
+                {
+                    head = queue.peek();
+                    if (canRun(head))
+                        head.onUnblocked();
+                }
+            }
+            return operationToConflicts.remove(operation);
+        }
+
+        public Conflicts conflicts(AsyncOperation<?> operation)
+        {
+            return operationToConflicts.get(operation);
+        }
+    }
+
     private final Object2ObjectHashMap<Object, Object> queues = new Object2ObjectHashMap<>();
+    private final RangeTree<RoutingKey, Range, RangeState> rangeQueues = RTree.create(RangeTreeRangeAccessor.instance);
 
     /**
      * Register an operation as having a dependency on its keys and TxnIds
@@ -39,14 +126,88 @@ public class ExecutionOrder
     boolean register(AsyncOperation<?> operation)
     {
         boolean canRun = true;
-        for (RoutableKey key : operation.keys())
-            canRun &= register(key, operation);
+        for (Seekable seekable : operation.keys())
+        {
+            switch (seekable.domain())
+            {
+                case Key:
+                    canRun &= register(seekable.asKey(), operation);
+                    break;
+                case Range:
+                    canRun &= register(seekable.asRange(), operation);
+                    break;
+                default:
+                    throw new AssertionError("Unexpected domain: " + seekable.domain());
+            }
+        }
         TxnId primaryTxnId = operation.primaryTxnId();
         if (null != primaryTxnId)
             canRun &= register(primaryTxnId, operation);
         return canRun;
     }
 
+    private boolean register(Range range, AsyncOperation<?> operation)
+    {
+        // Ranges depend on Ranges and Keys
+        // Keys depend on Keys...
+        // This adds a complication to this logic as keys should be able to make progress regardless of ranges, but rangest must depend on keys
+        List<Key> keyConflicts = null;
+        for (Object o : queues.keySet())
+        {
+            if (!(o instanceof Key))
+                continue;
+            Key key = (Key) o;
+            if (!range.contains(key))
+                continue;
+            if (keyConflicts == null)
+                keyConflicts = new ArrayList<>();
+            keyConflicts.add(key);
+        }
+        if (keyConflicts != null)
+            keyConflicts.forEach(k -> register(k, operation));
+
+        class Result
+        {
+            RangeState sameRange = null;
+            List<Range> rangeConflicts = null;
+        }
+        Result result = new Result();
+        rangeQueues.search(range, e -> {
+            if (range.equals(e.getKey()))
+                result.sameRange = e.getValue();
+            else
+            {
+                if (result.rangeConflicts == null)
+                    result.rangeConflicts = new ArrayList<>();
+                result.rangeConflicts.add(e.getKey());
+            }
+            RangeState state = e.getValue();
+            Object operationOrQueue = state.operationOrQueue;
+            if (operationOrQueue instanceof AsyncOperation)
+            {
+                ArrayDeque<AsyncOperation<?>> queue = new ArrayDeque<>(4);
+                queue.add((AsyncOperation<?>) operationOrQueue);
+                queue.add(operation);
+                state.operationOrQueue = queue;
+            }
+            else
+            {
+                @SuppressWarnings("unchecked")
+                ArrayDeque<AsyncOperation<?>> queue = (ArrayDeque<AsyncOperation<?>>) operationOrQueue;
+                queue.add(operation);
+            }
+        });
+        if (result.sameRange != null)
+        {
+            result.sameRange.add(operation, keyConflicts, result.rangeConflicts);
+        }
+        else
+        {
+            rangeQueues.add(range, new RangeState(range, keyConflicts, result.rangeConflicts, operation));
+        }
+        return keyConflicts == null && result.rangeConflicts == null;
+    }
+
     /**
      * Register an operation as having a dependency on a key or a TxnId
      * @return true if no other operation depends on the key/TxnId, false otherwise
@@ -81,13 +242,36 @@ public class ExecutionOrder
      */
     void unregister(AsyncOperation<?> operation)
     {
-        for (RoutableKey key : operation.keys())
-            unregister(key, operation);
+        for (Seekable seekable : operation.keys())
+        {
+            switch (seekable.domain())
+            {
+                case Key:
+                    unregister(seekable.asKey(), operation);
+                    break;
+                case Range:
+                    unregister(seekable.asRange(), operation);
+                    break;
+                default:
+                    throw new AssertionError("Unexpected domain: " + seekable.domain());
+            }
+
+        }
         TxnId primaryTxnId = operation.primaryTxnId();
         if (null != primaryTxnId)
             unregister(primaryTxnId, operation);
     }
 
+    private void unregister(Range range, AsyncOperation<?> operation)
+    {
+        RangeState state = state(range);
+        Conflicts conflicts = state.remove(operation);
+        if (conflicts.rangeConflicts != null)
+            conflicts.rangeConflicts.forEach(r -> state(r).remove(operation));
+        if (conflicts.keyConflicts != null)
+            conflicts.keyConflicts.forEach(k -> unregister(k, operation));
+    }
+
     /**
      * Unregister the operation as being a dependency for key or TxnId
      */
@@ -123,14 +307,60 @@ public class ExecutionOrder
 
     boolean canRun(AsyncOperation<?> operation)
     {
-        for (RoutableKey key : operation.keys())
-            if (!canRun(key, operation))
-                return false;
+        for (Seekable seekable : operation.keys())
+        {
+            switch (seekable.domain())
+            {
+                case Key:
+                    if (!canRun(seekable.asKey(), operation))
+                        return false;
+                    break;
+                case Range:
+                    if (!canRun(seekable.asRange(), operation))
+                        return false;
+                    break;
+                default:
+                    throw new AssertionError("Unexpected domain: " + seekable.domain());
+            }
+
+        }
 
         TxnId primaryTxnId = operation.primaryTxnId();
         return primaryTxnId == null || canRun(primaryTxnId, operation);
     }
 
+    private boolean canRun(Range range, AsyncOperation<?> operation)
+    {
+        RangeState state = state(range);
+        if (!state.canRun(operation))
+            return false;
+        Conflicts conflicts = state.conflicts(operation);
+        if (conflicts.rangeConflicts != null)
+        {
+            for (Range r : conflicts.rangeConflicts)
+            {
+                if (!state(r).canRun(operation))
+                    return false;
+            }
+        }
+        if (conflicts.keyConflicts != null)
+        {
+            for (Key key : conflicts.keyConflicts)
+            {
+                if (!canRun(key, operation))
+                    return false;
+            }
+        }
+        return true;
+    }
+
+    private RangeState state(Range range)
+    {
+        List<RangeState> list = rangeQueues.get(range);
+        assert list.size() == 1 : String.format("Expected 1 element but saw list %s", list);
+        return list.get(0);
+    }
+
     private boolean canRun(Object keyOrTxnId, AsyncOperation<?> operation)
     {
         Object operationOrQueue = queues.get(keyOrTxnId);

src/java/org/apache/cassandra/service/accord/events/CacheEvents.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.events;
+
+import jdk.jfr.Category;
+import jdk.jfr.DataAmount;
+import jdk.jfr.Event;
+import jdk.jfr.Label;
+import jdk.jfr.Name;
+import jdk.jfr.Percentage;
+import jdk.jfr.StackTrace;
+
+@Category({"Accord", "Accord Cache"})
+@StackTrace(false)
+public abstract class CacheEvents extends Event
+{
+    public int store;
+    public String instance;
+    public String key;
+    public String status;
+    @DataAmount(DataAmount.BYTES)
+    public int lastQueriedEstimatedSizeOnHeap;
+
+    // instance
+    @DataAmount(DataAmount.BYTES)
+    public long instanceAllocated;
+    public long instanceStatsQueries, instanceStatsHits, instanceStatsMisses;
+
+    @Percentage
+    public double instanceStatsHitRate;
+
+    // cache
+    @DataAmount(DataAmount.BYTES)
+    public long globalCapacity, globalAllocated;
+    public int globalSize, globalReferenced, globalUnreferenced;
+
+    public long globalStatsQueries, globalStatsHits, globalStatsMisses;
+
+    @Percentage
+    public double globalStatsHitRate;
+
+    @Percentage
+    public double globalFree;
+    public void update()
+    {
+        instanceStatsHitRate =  1D - (instanceStatsHits / (double) instanceStatsQueries);
+        globalStatsHitRate =  1D - (globalStatsHits / (double) globalStatsQueries);
+        globalFree =  1.0D - (globalAllocated / (double) globalCapacity);
+    }
+
+    @Name("cassandra.accord.cache.Add")
+    @Label("Accord Cache Add")
+    public static class Add extends CacheEvents { }
+
+    @Name("cassandra.accord.cache.Release")
+    @Label("Accord Cache Release")
+    public static class Release extends CacheEvents { }
+
+    @Name("cassandra.accord.cache.Evict")
+    @Label("Accord Cache Evict")
+    public static class Evict extends CacheEvents { }
+}

src/java/org/apache/cassandra/service/accord/serializers/AccordRoutingKeyByteSource.java

@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.function.Function;
+
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.ValueAccessor;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+import static org.apache.cassandra.service.accord.api.AccordRoutingKey.RoutingKeyKind.SENTINEL;
+
+public class AccordRoutingKeyByteSource
+{
+    private static final byte[] MIN_ORDER = { -1 };
+    private static final byte[] TOKEN_ORDER = { 0 };
+    private static final byte[] MAX_ORDER = { 1 };
+
+    private static ByteSource minPrefix()
+    {
+        return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MIN_ORDER);
+    }
+
+    private static ByteSource tokenPrefix()
+    {
+        return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, TOKEN_ORDER);
+    }
+
+    private static ByteSource maxPrefix()
+    {
+        return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MAX_ORDER);
+    }
+
+    public static Serializer create(IPartitioner partitioner)
+    {
+        if (partitioner.isFixedLength())
+            return new FixedLength(partitioner, ByteComparable.Version.OSS50);
+        return new VariableLength(partitioner, ByteComparable.Version.OSS50);
+    }
+
+    public static FixedLength fixedLength(IPartitioner partitioner)
+    {
+        return new FixedLength(partitioner, ByteComparable.Version.OSS50);
+    }
+
+    public static VariableLength variableLength(IPartitioner partitioner)
+    {
+        return new VariableLength(partitioner, ByteComparable.Version.OSS50);
+    }
+
+    public static abstract class Serializer
+    {
+        protected final IPartitioner partitioner;
+        protected final ByteComparable.Version version;
+        protected final byte[] empty;
+
+        protected Serializer(IPartitioner partitioner, ByteComparable.Version version, byte[] empty)
+        {
+            this.partitioner = partitioner;
+            this.version = version;
+            this.empty = empty;
+        }
+
+        public ByteSource minAsComparableBytes()
+        {
+            return ByteSource.withTerminator(ByteSource.TERMINATOR, minPrefix(), ByteSource.fixedLength(empty));
+        }
+
+        public ByteSource maxAsComparableBytes()
+        {
+            return ByteSource.withTerminator(ByteSource.TERMINATOR, maxPrefix(), ByteSource.fixedLength(empty));
+        }
+
+        public ByteSource asComparableBytes(Token token)
+        {
+            if (token.getPartitioner() != partitioner)
+                throw new IllegalArgumentException("Attempted to use the wrong partitioner: given " + token.getPartitioner() + " but expected " + partitioner);
+            return ByteSource.withTerminator(ByteSource.TERMINATOR, tokenPrefix(), token.asComparableBytes(version));
+        }
+
+        public <V> Token tokenFromComparableBytes(ValueAccessor<V> accessor, V data) throws IOException
+        {
+            return tokenFromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(accessor, data)));
+        }
+
+        public Token tokenFromComparableBytes(ByteSource.Peekable bs) throws IOException
+        {
+            if (bs.peek() == ByteSource.TERMINATOR)
+                throw new IOException("Unable to read prefix");
+            ByteSource.Peekable component = progress(bs);
+
+            byte[] prefix = ByteSourceInverse.getOptionalSignedFixedLength(ByteArrayAccessor.instance, component, 1);
+            if (prefix == null)
+                throw new IOException("Unable to read prefix; prefix was null");
+            if (!Arrays.equals(TOKEN_ORDER, prefix))
+            {
+                String match = Arrays.equals(MIN_ORDER, prefix) ? "min"
+                                                                : Arrays.equals(MAX_ORDER, prefix) ? "max"
+                                                                                                   : "unknown";
+                throw new IOException("Attempt to read token from non-token value: was " + match);
+            }
+            component = ByteSourceInverse.nextComponentSource(bs);
+            if (component == null)
+                throw new IOException("Unable to read token; component was not found");
+            return partitioner.getTokenFactory().fromComparableBytes(component, version);
+        }
+
+        public ByteSource asComparableBytes(AccordRoutingKey key)
+        {
+            UUID uuid = key.table().asUUID();
+            ByteSource[] srcs = { LongType.instance.asComparableBytes(LongType.instance.decompose(uuid.getMostSignificantBits()), ByteComparable.Version.OSS50),
+                                  LongType.instance.asComparableBytes(LongType.instance.decompose(uuid.getLeastSignificantBits()), ByteComparable.Version.OSS50),
+                                  asComparableBytesNoTable(key) };
+            return ByteSource.withTerminator(ByteSource.TERMINATOR, srcs);
+        }
+
+        public ByteSource asComparableBytesNoTable(AccordRoutingKey key)
+        {
+            return key.kindOfRoutingKey() == SENTINEL ? key.asSentinelKey().isMin ? minAsComparableBytes() : maxAsComparableBytes()
+                                                      : asComparableBytes(key.token());
+        }
+
+        public <V> AccordRoutingKey fromComparableBytes(ValueAccessor<V> accessor, V data) throws IOException
+        {
+            ByteSource.Peekable bs = ByteSource.peekable(ByteSource.fixedLength(accessor, data));
+            long[] uuidValues = new long[2];
+            for (int i = 0; i < 2; i++)
+            {
+                if (bs.peek() == ByteSource.TERMINATOR)
+                    throw new IllegalArgumentException("Unable to parse bytes");
+                ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(bs);
+                long value = LongType.instance.compose(LongType.instance.fromComparableBytes(component, ByteComparable.Version.OSS50));
+                uuidValues[i] = value;
+            }
+            TableId tableId = TableId.fromUUID(new UUID(uuidValues[0], uuidValues[1]));
+            return fromComparableBytes(bs,
+                                       isMin -> isMin ? AccordRoutingKey.SentinelKey.min(tableId) : AccordRoutingKey.SentinelKey.max(tableId),
+                                       token -> new AccordRoutingKey.TokenKey(tableId, token));
+        }
+
+        private AccordRoutingKey fromComparableBytes(ByteSource.Peekable bs,
+                                                     Function<Boolean, AccordRoutingKey> onSentinel,
+                                                     Function<Token, AccordRoutingKey> onToken) throws IOException
+        {
+            if (bs.peek() == ByteSource.TERMINATOR)
+                throw new IOException("Unable to read prefix");
+            ByteSource.Peekable component = progress(bs);
+
+            byte[] prefix = ByteSourceInverse.getOptionalSignedFixedLength(ByteArrayAccessor.instance, component, 1);
+            if (prefix == null)
+                throw new IOException("Unable to read prefix; prefix was null");
+            if (Arrays.equals(TOKEN_ORDER, prefix))
+            {
+                component = ByteSourceInverse.nextComponentSource(bs);
+                if (component == null)
+                    throw new IOException("Unable to read token; component was not found");
+                return onToken.apply(partitioner.getTokenFactory().fromComparableBytes(component, version));
+            }
+            if (Arrays.equals(MIN_ORDER, prefix))
+                return onSentinel.apply(true);
+            if (Arrays.equals(MAX_ORDER, prefix))
+                return onSentinel.apply(false);
+            throw new AssertionError("Unknown prefix");
+        }
+
+        private static ByteSource.Peekable progress(ByteSource.Peekable bs) throws IOException
+        {
+            ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(bs);
+            if (component == null)
+                throw new IOException("Unable to read prefix; component was not found");
+            if (component.peek() == ByteSource.NEXT_COMPONENT)
+            {
+                // this came from (table, token_or_sentinel)
+                component = ByteSourceInverse.nextComponentSource(bs);
+                if (component == null)
+                    throw new IOException("Unable to read prefix; component was not found");
+            }
+            return component;
+        }
+
+        public byte[] serialize(Token token)
+        {
+            return ByteSourceInverse.readBytes(asComparableBytes(token));
+        }
+
+        public byte[] serialize(AccordRoutingKey key)
+        {
+            return ByteSourceInverse.readBytes(asComparableBytes(key));
+        }
+
+        public byte[] serializeNoTable(AccordRoutingKey key)
+        {
+            return ByteSourceInverse.readBytes(asComparableBytesNoTable(key));
+        }
+    }
+
+    public static class VariableLength extends Serializer
+    {
+        public VariableLength(IPartitioner partitioner, ByteComparable.Version version)
+        {
+            super(partitioner, version, ByteArrayUtil.EMPTY_BYTE_ARRAY);
+        }
+    }
+
+    public static class FixedLength extends Serializer
+    {
+        public FixedLength(IPartitioner partitioner, ByteComparable.Version version)
+        {
+            super(partitioner, version, computeEmptyBytes(partitioner, version));
+        }
+
+        private static byte[] computeEmptyBytes(IPartitioner partitioner, ByteComparable.Version version)
+        {
+            if (!partitioner.isFixedLength())
+                throw new IllegalArgumentException("Unable to use partitioner " + partitioner.getClass() + "; it is not fixed-length");
+
+            int tokenSize = ByteSourceInverse.readBytes(partitioner.getMinimumToken().asComparableBytes(version)).length;
+            return new byte[tokenSize];
+        }
+
+        public int valueSize()
+        {
+            return 4 + empty.length;
+        }
+    }
+}

src/java/org/apache/cassandra/utils/Clock.java

@@ -107,6 +107,14 @@ public interface Clock
         {
             return instance.currentTimeMillis();
         }
+
+        /**
+         * Semantically equivalent to {@link FBUtilities#nowInSeconds()}
+         */
+        public static long nowInSeconds()
+        {
+            return instance.nowInSeconds();
+        }
     }
 
     public static class Default implements Clock

src/java/org/apache/cassandra/utils/CloseableIterator.java

@@ -66,5 +66,4 @@ public interface CloseableIterator<T> extends Iterator<T>, AutoCloseable
             }
         };
     }
-
 }

src/java/org/apache/cassandra/utils/IntervalTree.java

@@ -28,6 +28,8 @@ import java.util.List;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
@@ -187,6 +189,11 @@ public class IntervalTree<C extends Comparable<? super C>, D extends Comparable<
         return search(Interval.<C, D>create(point, point, null));
     }
 
+    public List<D> search(C start, C end)
+    {
+        return search(Interval.<C, D>create(start, end, null));
+    }
+
     /**
      * The input arrays aren't defensively copied and will be sorted. The update method doesn't allow duplicates or elements to be removed
      * to be missing and this differs from the constructor which does not duplicate checking at all.
@@ -312,6 +319,11 @@ public class IntervalTree<C extends Comparable<? super C>, D extends Comparable<
         return new TreeIterator(head);
     }
 
+    public Stream<I> stream()
+    {
+        return StreamSupport.stream(spliterator(), false);
+    }
+
     @Override
     public String toString()
     {
@@ -555,11 +567,6 @@ public class IntervalTree<C extends Comparable<? super C>, D extends Comparable<
             return this;
         }
 
-        public interface TriPredicate<A, B, C>
-        {
-            boolean test(A a, B b, C c);
-        }
-
         public Builder<C, D, I> removeIf(TriPredicate<C, C, D> predicate)
         {
             intervals.removeIf(i -> predicate.test(i.min, i.max, i.data));

src/java/org/apache/cassandra/utils/MutableEntry.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Map;
+import java.util.Objects;
+
+public class MutableEntry<K, V> implements Map.Entry<K, V>
+{
+    private final K k;
+    private V v;
+
+    public MutableEntry(K k, V v)
+    {
+        this.k = k;
+        this.v = v;
+    }
+
+    @Override
+    public K getKey()
+    {
+        return k;
+    }
+
+    @Override
+    public V getValue()
+    {
+        return v;
+    }
+
+    @Override
+    public V setValue(V value)
+    {
+        V previous = v;
+        v = Objects.requireNonNull(value);
+        return previous;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || !(o instanceof Map.Entry)) return false;
+        Map.Entry<?, ?> that = (Map.Entry<?, ?>) o;
+        return Objects.equals(k, that.getKey()) && Objects.equals(v, that.getValue());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(k, v);
+    }
+
+    @Override
+    public String toString()
+    {
+        return k + "=" + v;
+    }
+}

src/java/org/apache/cassandra/utils/RTree.java

@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import javax.annotation.CheckForNull;
+
+import com.google.common.collect.AbstractIterator;
+
+public class RTree<Token, Range, Value> implements RangeTree<Token, Range, Value>
+{
+    /**
+     * Tuning size target can be tricky as it is based on expected access patterns and expected matche sizes.  There is also
+     * a memory cost to account for as large tree sizes will have far more nodes with a small target than a large target.
+     *
+     * If matching most of the data then larger sizes leads to fewer hops
+     * If matching few elements then tree depth maters the most, if walking a long tree is more costly than walking the
+     * element list, then shrinking depth (by having larger size target) can improve performance.
+     */
+    private static final int DEFAULT_SIZE_TARGET = 1 << 7;
+    private static final int DEFAULT_NUMBER_OF_CHILDREN = 6;
+
+    private final Comparator<Token> comparator;
+    private final Accessor<Token, Range> accessor;
+    private final int sizeTarget;
+    private final int numChildren;
+    private Node node = new Node();
+
+    public RTree(Comparator<Token> comparator, Accessor<Token, Range> accessor)
+    {
+        this(comparator, accessor, DEFAULT_SIZE_TARGET, DEFAULT_NUMBER_OF_CHILDREN);
+    }
+
+    public RTree(Comparator<Token> comparator, Accessor<Token, Range> accessor, int sizeTarget, int numChildren)
+    {
+        if (sizeTarget <= 1)
+            throw new IllegalArgumentException("size target must be 2 or more");
+        if (numChildren <= 1)
+            throw new IllegalArgumentException("Number of children must be 2 or more");
+        if (sizeTarget < numChildren)
+            throw new IllegalArgumentException("Size target (" + sizeTarget + ") was less than number of children (" + numChildren + ")");
+        this.comparator = comparator;
+        this.accessor = accessor;
+        this.sizeTarget = sizeTarget;
+        this.numChildren = numChildren;
+    }
+
+    public static <Token extends Comparable<? super Token>, Range, Value> RTree<Token, Range, Value> create(Accessor<Token, Range> accessor)
+    {
+        return new RTree<>(Comparator.naturalOrder(), accessor);
+    }
+
+    @Override
+    public List<Value> get(Range range)
+    {
+        List<Value> matches = new ArrayList<>();
+        get(range, e -> matches.add(e.getValue()));
+        return matches;
+    }
+
+    @Override
+    public void get(Range range, Consumer<Map.Entry<Range, Value>> onMatch)
+    {
+        node.search(range, onMatch, e -> e.getKey().equals(range), Function.identity());
+    }
+
+    @Override
+    public List<Map.Entry<Range, Value>> search(Range range)
+    {
+        List<Map.Entry<Range, Value>> matches = new ArrayList<>();
+        search(range, matches::add);
+        return matches;
+    }
+
+    @Override
+    public void search(Range range, Consumer<Map.Entry<Range, Value>> onMatch)
+    {
+        node.search(range, onMatch, ignore -> true, Function.identity());
+    }
+
+    public List<Value> find(Range range)
+    {
+        List<Value> matches = new ArrayList<>();
+        find(range, matches::add);
+        return matches;
+    }
+
+    public void find(Range range, Consumer<Value> onMatch)
+    {
+        node.search(range, onMatch, ignore -> true, Map.Entry::getValue);
+    }
+
+    @Override
+    public List<Map.Entry<Range, Value>> searchToken(Token token)
+    {
+        List<Map.Entry<Range, Value>> matches = new ArrayList<>();
+        searchToken(token, matches::add);
+        return matches;
+    }
+
+    @Override
+    public void searchToken(Token token, Consumer<Map.Entry<Range, Value>> onMatch)
+    {
+        node.searchToken(token, onMatch, ignore -> true, Function.identity());
+    }
+
+    public List<Value> findToken(Token token)
+    {
+        List<Value> matches = new ArrayList<>();
+        findToken(token, matches::add);
+        return matches;
+    }
+
+    public void findToken(Token token, Consumer<Value> onMatch)
+    {
+        node.searchToken(token, onMatch, ignore -> true, Map.Entry::getValue);
+    }
+
+    @Override
+    public boolean add(Range key, Value value)
+    {
+        node.add(key, value);
+        return true;
+    }
+
+    @Override
+    public int remove(Range key)
+    {
+        return node.removeIf(e -> e.getKey().equals(key));
+    }
+
+    public int remove(Range key, Value value)
+    {
+        Map.Entry<Range, Value> match = Map.entry(key, value);
+        return node.removeIf(match::equals);
+    }
+
+    @Override
+    public void clear()
+    {
+        node = new Node();
+    }
+
+    @Override
+    public int size()
+    {
+        return node.size;
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return node.size == 0;
+    }
+
+    public String displayTree()
+    {
+        StringBuilder sb = new StringBuilder();
+        node.displayTree(0, sb);
+        return sb.toString();
+    }
+
+    @Override
+    public Iterator<Map.Entry<Range, Value>> iterator()
+    {
+        return node.iterator();
+    }
+
+    @Override
+    public Stream<Map.Entry<Range, Value>> stream()
+    {
+        return StreamSupport.stream(spliterator(), false);
+    }
+
+    private class Node implements Iterable<Map.Entry<Range, Value>>
+    {
+        private List<Map.Entry<Range, Value>> values = new ArrayList<>();
+        private List<Node> children = null;
+        private int size = 0;
+        private Token minStart, maxStart, minEnd, maxEnd;
+
+        int removeIf(Predicate<Map.Entry<Range, Value>> condition)
+        {
+            if (minStart == null)
+                return 0;
+            if (children != null)
+            {
+                int sum = 0;
+                for (Node node : children)
+                    sum += node.removeIf(condition);
+                size -= sum;
+                return sum;
+            }
+            class Counter {int value;}
+            Counter counter = new Counter();
+            values.removeIf(e -> {
+                if (condition.test(e))
+                {
+                    counter.value++;
+                    return true;
+                }
+                return false;
+            });
+            size -= counter.value;
+            if (values.isEmpty())
+                minStart = maxStart = minEnd = maxEnd = null;
+            return counter.value;
+        }
+
+        void add(Range range, Value value)
+        {
+            size++;
+            if (minStart == null)
+            {
+                minStart = maxStart = accessor.start(range);
+                minEnd = maxEnd = accessor.end(range);
+            }
+            else
+            {
+                Token start = accessor.start(range);
+                minStart = min(minStart, start);
+                maxStart = max(maxStart, start);
+                Token end = accessor.end(range);
+                minEnd = min(minEnd, end);
+                maxEnd = max(maxEnd, end);
+            }
+            if (children != null)
+            {
+                findBestMatch(range).add(range, value);
+                return;
+            }
+            values.add(new MutableEntry(range, value));
+            if (shouldSplit())
+                split();
+        }
+
+        private Node findBestMatch(Range range)
+        {
+            int topIdx = 0;
+            Node node = children.get(0);
+            int topScore = node.score(range);
+            int size = node.size;
+            for (int i = 1; i < children.size(); i++)
+            {
+                node = children.get(i);
+                int score = node.score(range);
+                if (score > topScore || (score == topScore && size > node.size))
+                {
+                    topIdx = i;
+                    size = node.size;
+                }
+            }
+            return children.get(topIdx);
+        }
+
+        private int score(Range range)
+        {
+            if (minStart == null)
+                return 0;
+            if (!intersects(range))
+                return -10;
+            int score = 5; // overlapps
+            if (values != null) // is leaf
+                score += 5;
+
+            int startScore = 0;
+            if (comparator.compare(maxStart, accessor.start(range)) <= 0)
+                startScore += 10;
+            else if (comparator.compare(minStart, accessor.start(range)) <= 0)
+                startScore += 5;
+
+            int endScore = 0;
+            if (comparator.compare(minEnd, accessor.end(range)) >= 0)
+                endScore += 10;
+            else if (comparator.compare(maxEnd, accessor.end(range)) >= 0)
+                endScore += 5;
+            // if fully contained, then add the scores: 10 for largest bounds, 20 for smallest bounds
+            if (!(startScore == 0 || endScore == 0))
+                score += startScore + endScore;
+            return score;
+        }
+
+        boolean shouldSplit()
+        {
+            return values.size() > sizeTarget
+                   // if the same range is used over and over again, splitting doesn't do much
+                   && !(comparator.compare(minStart, maxStart) == 0
+                        && comparator.compare(minEnd, maxEnd) == 0);
+        }
+
+        List<List<Map.Entry<Range, Value>>> partitionByEnd()
+        {
+            List<Token> allEndpoints = new ArrayList<>(values.size() * 2);
+            for (Map.Entry<Range, Value> a : values)
+            {
+                allEndpoints.add(accessor.start(a.getKey()));
+                allEndpoints.add(accessor.end(a.getKey()));
+            }
+            allEndpoints.sort(comparator);
+            List<Token> maxToken = new ArrayList<>(numChildren);
+            int tick = allEndpoints.size() / numChildren;
+            int offset = tick;
+            for (int i = 0; i < numChildren; i++)
+            {
+                maxToken.add(allEndpoints.get(offset));
+                offset += tick;
+                if (offset >= allEndpoints.size())
+                {
+                    maxToken.add(allEndpoints.get(allEndpoints.size() - 1));
+                    break;
+                }
+            }
+
+            List<List<Map.Entry<Range, Value>>> partitions = new ArrayList<>(numChildren);
+            for (int i = 0; i < numChildren; i++)
+                partitions.add(new ArrayList<>());
+
+            for (Map.Entry<Range, Value> a : values)
+            {
+                Token end = accessor.end(a.getKey());
+                List<Map.Entry<Range, Value>> selected = null;
+                for (int i = 0; i < numChildren; i++)
+                {
+                    if (comparator.compare(end, maxToken.get(i)) < 0)
+                    {
+                        selected = partitions.get(i);
+                        break;
+                    }
+                }
+                if (selected == null)
+                    selected = partitions.get(partitions.size() - 1);
+                selected.add(a);
+            }
+            int[] sizes = partitions.stream().mapToInt(List::size).toArray();
+            return goodEnough(sizes) ? partitions : null;
+        }
+
+        private boolean goodEnough(int[] sizes)
+        {
+            double sum = 0.0;
+            for (int i : sizes)
+                sum += i;
+            double mean = sum / sizes.length;
+            double stddev = 0.0;
+            for (int i : sizes)
+                stddev += Math.pow(i - mean, 2);
+            stddev = Math.sqrt(stddev / sizes.length);
+            return stddev < 1.5;
+        }
+
+        void split()
+        {
+            children = new ArrayList<>(numChildren);
+            for (int i = 0; i < numChildren; i++)
+                children.add(new Node());
+
+            List<List<Map.Entry<Range, Value>>> partitions = partitionByEnd();
+            if (partitions == null)
+                partitions = partitionEven();
+            for (int i = 0; i < children.size(); i++)
+            {
+                Node c = children.get(i);
+                List<Map.Entry<Range, Value>> entries = partitions.get(i);
+                entries.forEach(e -> c.add(e.getKey(), e.getValue()));
+            }
+
+            values.clear();
+            values = null;
+        }
+
+        private List<List<Map.Entry<Range, Value>>> partitionEven()
+        {
+            values.sort((a, b) -> {
+                Range left = a.getKey();
+                Range right = b.getKey();
+                int rc = comparator.compare(accessor.start(left), accessor.start(right));
+                if (rc == 0)
+                    rc = comparator.compare(accessor.end(left), accessor.end(right));
+                return rc;
+            });
+            List<List<Map.Entry<Range, Value>>> partition = new ArrayList<>(numChildren);
+            int size = Math.max(1, values.size() / numChildren);
+            int offset = 0;
+            for (int i = 0; i < numChildren - 1; i++)
+            {
+                int total = size;
+                partition.add(new ArrayList<>(values.subList(offset, offset + total)));
+                offset += total;
+            }
+            partition.add(new ArrayList<>(values.subList(offset, values.size())));
+            return partition;
+        }
+
+        <T> void search(Range range, Consumer<T> matches, Predicate<Map.Entry<Range, Value>> predicate, Function<Map.Entry<Range, Value>, T> transformer)
+        {
+            if (minStart == null)
+                return;
+            if (!intersects(range))
+                return;
+            if (children != null)
+            {
+                children.forEach(n -> n.search(range, matches, predicate, transformer));
+                return;
+            }
+            values.forEach(e -> {
+                if (accessor.intersects(e.getKey(), range) && predicate.test(e))
+                    matches.accept(transformer.apply(e));
+            });
+        }
+
+        <T> void searchToken(Token token, Consumer<T> matches, Predicate<Map.Entry<Range, Value>> predicate, Function<Map.Entry<Range, Value>, T> transformer)
+        {
+            if (minStart == null)
+                return;
+            if (!contains(minStart, maxEnd, token))
+                return;
+            if (children != null)
+            {
+                for (int i = 0, size = children.size(); i < size; i++)
+                {
+                    Node node = children.get(i);
+                    node.searchToken(token, matches, predicate, transformer);
+                }
+                return;
+            }
+            values.forEach(e -> {
+                if (accessor.contains(e.getKey(), token) && predicate.test(e))
+                    matches.accept(transformer.apply(e));
+            });
+        }
+
+        boolean intersects(Range range)
+        {
+            return accessor.intersects(range, minStart, maxEnd);
+        }
+
+        boolean contains(Token start, Token end, Token value)
+        {
+            return accessor.contains(start, end, value);
+        }
+
+        private void displayTree(int level, StringBuilder sb)
+        {
+            for (int i = 0; i < level; i++)
+                sb.append('\t');
+            sb.append("start:(").append(minStart).append(", ").append(maxStart).append("), end:(").append(minEnd).append(", ").append(maxEnd).append("):");
+            if (children != null)
+            {
+                sb.append('\n');
+                children.forEach(n -> n.displayTree(level + 1, sb));
+            }
+            else
+            {
+                sb.append(' ').append(size).append('\n');
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return "Node{" +
+                   "minStart=" + minStart +
+                   ", maxStart=" + maxStart +
+                   ", minEnd=" + minEnd +
+                   ", maxEnd=" + maxEnd +
+                   ", values=" + values +
+                   ", children=" + children +
+                   '}';
+        }
+
+        private Token min(Token a, Token b)
+        {
+            return comparator.compare(a, b) < 0 ? a : b;
+        }
+
+        private Token max(Token a, Token b)
+        {
+            return comparator.compare(a, b) < 0 ? b : a;
+        }
+
+        @Override
+        public Iterator<Map.Entry<Range, Value>> iterator()
+        {
+            if (values != null)
+                return values.iterator();
+            return new AbstractIterator<>()
+            {
+                private int index = 0;
+                private Iterator<Map.Entry<Range, Value>> it = null;
+                @CheckForNull
+                @Override
+                protected Map.Entry<Range, Value> computeNext()
+                {
+                    while (true)
+                    {
+                        if (it == null)
+                        {
+                            if (index == children.size())
+                                return endOfData();
+                            it = children.get(index++).iterator();
+                        }
+                        if (it.hasNext())
+                            return it.next();
+                        it = null;
+                    }
+                }
+            };
+        }
+    }
+}

src/java/org/apache/cassandra/utils/RangeTree.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public interface RangeTree<Token, Range, Value> extends Iterable<Map.Entry<Range, Value>>
+{
+    void searchToken(Token token, Consumer<Map.Entry<Range, Value>> onMatch);
+
+    boolean add(Range key, Value value);
+
+    List<Value> get(Range range);
+
+    void get(Range range, Consumer<Map.Entry<Range, Value>> onMatch);
+
+    List<Map.Entry<Range, Value>> search(Range range);
+
+    void search(Range range, Consumer<Map.Entry<Range, Value>> onMatch);
+
+    List<Map.Entry<Range, Value>> searchToken(Token token);
+
+    int remove(Range key);
+
+    void clear();
+
+    int size();
+
+    boolean isEmpty();
+
+    default Stream<Map.Entry<Range, Value>> stream()
+    {
+        return StreamSupport.stream(spliterator(), false);
+    }
+
+    interface Accessor<Token, Range>
+    {
+        Token start(Range range);
+        Token end(Range range);
+        boolean contains(Token start, Token end, Token token);
+        default boolean contains(Range range, Token token)
+        {
+            return contains(start(range), end(range), token);
+        }
+        boolean intersects(Range range, Token start, Token end);
+        default boolean intersects(Range left, Range right)
+        {
+            return intersects(left, start(right), end(right));
+        }
+    }
+}

src/java/org/apache/cassandra/utils/TriPredicate.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+public interface TriPredicate<A, B, C>
+{
+    boolean test(A a, B b, C c);
+}

test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java

@@ -41,6 +41,8 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.NetworkTopologyProximity;
 import org.apache.cassandra.locator.SimpleSeedProvider;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_ACCORD_ENABLED;
+
 public class InstanceConfig implements IInstanceConfig
 {
     public final int num;
@@ -323,7 +325,7 @@ public class InstanceConfig implements IInstanceConfig
     {
         int seedNode = provisionStrategy.seedNodeNum();
         AccordSpec accordSpec = new AccordSpec();
-        accordSpec.enabled = true;
+        accordSpec.enabled = DTEST_ACCORD_ENABLED.getBoolean();
         accordSpec.journal_directory = String.format("%s/node%d/accord_journal", root, nodeNum);
         accordSpec.shard_count = new OptionaldPositiveInt(4);
         return new InstanceConfig(nodeNum,

test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java

@@ -18,41 +18,14 @@
 
 package org.apache.cassandra.distributed.test.accord;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
-
+import accord.primitives.Unseekables;
+import accord.topology.Topologies;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.primitives.Unseekables;
-import accord.topology.Topologies;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.functions.types.utils.Bytes;
-import org.apache.cassandra.db.marshal.Int32Type;
-import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.MapType;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.ICoordinator;
@@ -64,18 +37,34 @@ import org.apache.cassandra.service.accord.AccordTestUtils;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.assertj.core.api.Assertions;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonList;
 import static org.apache.cassandra.cql3.CQLTester.row;
 import static org.apache.cassandra.distributed.util.QueryResultUtil.assertThat;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.*;
 
-@RunWith(Parameterized.class)
-public class AccordCQLTest extends AccordTestBase
+public abstract class AccordCQLTestBase extends AccordTestBase
 {
-    private static final Logger logger = LoggerFactory.getLogger(AccordCQLTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(AccordCQLTestBase.class);
+
+    private final TransactionalMode transactionalMode;
+
+    protected AccordCQLTestBase(TransactionalMode transactionalMode) {
+        this.transactionalMode = transactionalMode;
+    }
 
     @Override
     protected Logger logger()
@@ -83,24 +72,6 @@ public class AccordCQLTest extends AccordTestBase
         return logger;
     }
 
-    @Parameterized.Parameter
-    public String transactionalModeName;
-
-    TransactionalMode transactionalMode;
-
-    @Parameterized.Parameters(name = "transactionalMode={0}")
-    public static Collection<Object[]> data()
-    {
-        return ImmutableList.of(new Object[] {TransactionalMode.full.toString()},
-                                new Object[] {TransactionalMode.mixed_reads.toString()});
-    }
-
-    @Before
-    public void setNonSerialWriteStrategy()
-    {
-        transactionalMode = TransactionalMode.valueOf(transactionalModeName);
-    }
-
     @BeforeClass
     public static void setupClass() throws IOException
     {
@@ -2302,7 +2273,7 @@ public class AccordCQLTest extends AccordTestBase
     @Test
     public void testMultiCellMapSelection() throws Exception
     {
-        testMapSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map<text, int>)");
+        testMapSelection("CREATE TABLE " + qualifiedTableName + " (k int PRIMARY KEY, int_map map<text, int>) WITH transactional_mode='" + transactionalMode + "'");
     }
 
     @Test

test/distributed/org/apache/cassandra/distributed/test/accord/FullAccordCQLTest.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.accord;
+
+import org.apache.cassandra.service.consensus.TransactionalMode;
+
+public class FullAccordCQLTest extends AccordCQLTestBase
+{
+    public FullAccordCQLTest()
+    {
+        super(TransactionalMode.full);
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/accord/MixedReadAccordCQLTest.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.accord;
+
+import org.apache.cassandra.service.consensus.TransactionalMode;
+
+public class MixedReadAccordCQLTest extends AccordCQLTestBase
+{
+    public MixedReadAccordCQLTest()
+    {
+        super(TransactionalMode.mixed_reads);
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java

@@ -38,6 +38,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement;
 import org.apache.cassandra.cql3.statements.schema.KeyspaceAttributes;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.ByteOrderedPartitioner;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -51,6 +53,9 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.SchemaTransformation;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.tcm.AtomicLongBackedProcessor;
@@ -230,6 +235,30 @@ public class ClusterMetadataTestHelper
         }
     }
 
+    public static void setMemtable(String ks, String table, String memtable)
+    {
+        setMemtable(ks, table, MemtableParams.get(memtable));
+    }
+
+    public static void setMemtable(String ks, String table, MemtableParams memtable)
+    {
+        if (SchemaConstants.isLocalSystemKeyspace(ks))
+        {
+            ColumnFamilyStore store = Keyspace.open(ks).getColumnFamilyStore(table);
+            store.reload(store.metadata().unbuild().memtable(memtable).build());
+        }
+        else
+        {
+            Schema.instance.submit(cms -> {
+                var km = cms.schema.getKeyspaceMetadata(ks);
+                var update = km.withSwapped(km.tables.withSwapped(km.tables.getNullable(table).unbuild()
+                                                                     .memtable(memtable)
+                                                                     .build()));
+                return cms.schema.getKeyspaces().withAddedOrUpdated(update);
+            });
+        }
+    }
+
     private static Set<InetAddressAndPort> leaving(ClusterMetadata metadata)
     {
         return  metadata.directory.states.entrySet().stream()

test/distributed/org/apache/cassandra/distributed/upgrade/UpgradeTestBase.java

@@ -53,6 +53,7 @@ import org.apache.cassandra.distributed.shared.Versions;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.SimpleGraph;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_ACCORD_ENABLED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_GC_INSPECTOR;
 import static org.apache.cassandra.distributed.shared.Versions.Version;
 import static org.apache.cassandra.distributed.shared.Versions.find;
@@ -74,6 +75,7 @@ public class UpgradeTestBase extends DistributedTestBase
     {
         ICluster.setup();
         SKIP_GC_INSPECTOR.setBoolean(true);
+        DTEST_ACCORD_ENABLED.setBoolean(false);
     }
 
 

test/unit/accord/utilsfork/Gens.java

@@ -44,7 +44,7 @@ import java.util.stream.Stream;
 
 import com.google.common.collect.Iterables;
 
-import accord.utils.random.Picker;
+import accord.utilsfork.random.Picker;
 
 public class Gens {
     private Gens() {

test/unit/accord/utilsfork/RandomSource.java

@@ -33,7 +33,7 @@ import java.util.function.Supplier;
 
 import com.google.common.collect.Iterables;
 
-import accord.utils.random.Picker;
+import accord.utilsfork.random.Picker;
 
 // TODO (expected): merge with C* RandomSource
 public interface RandomSource
@@ -42,6 +42,11 @@ public interface RandomSource
     {
         return new accord.utilsfork.WrappedRandomSource(random);
     }
+    //TODO (maintaince): once the rebase is over remove this...
+    static RandomSource wrap(accord.utils.RandomSource rs)
+    {
+        return new WrappedRandomSource(rs.asJdkRandom());
+    }
 
     void nextBytes(byte[] bytes);
 

test/unit/accord/utilsfork/random/Picker.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package accord.utils.random;
+package accord.utilsfork.random;
 
 import java.util.Arrays;
 import java.util.function.Supplier;

test/unit/org/apache/cassandra/concurrent/ForwardingExecutorPlus.java

@@ -33,7 +33,7 @@ import org.apache.cassandra.utils.WithResources;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
 
-public class ForwardingExecutorPlus implements ExecutorPlus
+public class ForwardingExecutorPlus implements ExecutorPlus, SequentialExecutorPlus
 {
     private final ExecutorService delegate;
 
@@ -216,4 +216,10 @@ public class ForwardingExecutorPlus implements ExecutorPlus
         }
         throw new IllegalStateException("Unexpected future type: " + submit.getClass());
     }
+
+    @Override
+    public AtLeastOnceTrigger atLeastOnceTrigger(Runnable runnable)
+    {
+        return new SingleThreadExecutorPlus.AtLeastOnce(this, runnable);
+    }
 }

test/unit/org/apache/cassandra/cql3/CQLTester.java

@@ -2986,6 +2986,26 @@ public abstract class CQLTester
         throw new IllegalArgumentException("Unsupported value type (value is " + value + ")");
     }
 
+    protected static String wrapInTxn(String... stmts)
+    {
+        return wrapInTxn(Arrays.asList(stmts));
+    }
+
+    protected static String wrapInTxn(List<String> stmts)
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append("BEGIN TRANSACTION\n");
+        for (String stmt : stmts)
+        {
+            sb.append('\t').append(stmt);
+            if (!stmt.endsWith(";"))
+                sb.append(';');
+            sb.append('\n');
+        }
+        sb.append("COMMIT TRANSACTION");
+        return sb.toString();
+    }
+
     private static class TupleValue
     {
         protected final Object[] values;

test/unit/org/apache/cassandra/dht/IPartitionerTest.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.util.Objects;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.AbstractTypeGenerators;
+import org.apache.cassandra.utils.AccordGenerators;
+import org.apache.cassandra.utils.CassandraGenerators;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+
+public class IPartitionerTest
+{
+    @Test
+    public void byteCompareSerde()
+    {
+        qt().forAll(AccordGenerators.fromQT(CassandraGenerators.token())).check(token -> {
+            var p = token.getPartitioner();
+            var comparable = Objects.requireNonNull(ByteSource.peekable(p.getTokenFactory().asComparableBytes(token, ByteComparable.Version.OSS50)));
+            Token read = p.getTokenFactory().fromComparableBytes(comparable, ByteComparable.Version.OSS50);
+            Assertions.assertThat(read)
+                      .describedAs("If LocalPartitioner, the type is %s", (token.getPartitioner() instanceof LocalPartitioner ? AbstractTypeGenerators.typeTree(((LocalPartitioner) token.getPartitioner()).comparator) : null))
+                      .isEqualTo(token);
+        });
+    }
+}

test/unit/org/apache/cassandra/index/accord/AccordIndexStressTest.java

@@ -0,0 +1,517 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.api.RoutingKey;
+import accord.local.Node;
+import accord.local.SaveStatus;
+import accord.local.Status;
+import accord.primitives.FullKeyRoute;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable;
+import accord.primitives.Route;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.RandomSource;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.Long2ObjectHashMap;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.ReadSizeAbortException;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Interval;
+import org.apache.cassandra.utils.IntervalTree;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.utils.Clock.Global.nanoTime;
+
+/**
+ * This test validates the system_accord.commands.route index to make sure it returns the right values... the way the
+ * test is strucutred allows pluggability but runs with fixed configs while running in CI...
+ *
+ * If you are interested in testing different cases, the following tunables exist:
+ *
+ * <table>
+ *     <li>size: how many read/writes should be done</li>
+ *     <li>pattern: how ranges are layed out.  By default we test NO_OVERLAP so there are no conflicts</li>
+ * </table>
+ */
+public class AccordIndexStressTest extends CQLTester
+{
+    private static final Logger logger = LoggerFactory.getLogger(AccordIndexStressTest.class);
+
+    static
+    {
+        // The plan is to migrate away from SAI, so rather than hacking around timeout issues; just disable for now
+        CassandraRelevantProperties.SAI_TEST_DISABLE_TIMEOUT.setBoolean(true);
+    }
+    private static final boolean VALIDATE = true;
+    private static final boolean INCLUDE_FLUSH = true;
+    private static final long SLOW_NS = TimeUnit.MILLISECONDS.toNanos(25);
+    private static final Node.Id NODE = new Node.Id(42);
+    private final Routable.Domain domain = Routable.Domain.Range;
+    private final Int2ObjectHashMap<Map<TableId, Long2ObjectHashMap<List<TxnId>>>> storeToTableToRoutingKeysToTxns = new Int2ObjectHashMap<>();
+    private final Int2ObjectHashMap<Map<TableId, Map<TokenRange, List<TxnId>>>> storeToTableToRangesToTxns = new Int2ObjectHashMap<>();
+    private final RoutesSearcher searcher = new RoutesSearcher();
+
+    enum Size
+    {tiny, small, medium, large, benchmark}
+
+    private final Size size = Size.small;
+
+    private enum Read
+    {INDEX, CQL}
+
+    private Read read = Read.INDEX;
+
+    private enum Pattern { RANDOM, NO_OVERLAP }
+    private final Pattern pattern = Pattern.NO_OVERLAP;
+
+    @Test
+    public void test()
+    {
+        var tables = IntStream.range(0, 10)
+                              .mapToObj(i -> TableId.fromUUID(new UUID(0, i)))
+                              .collect(Collectors.toList());
+        int numWrites, numReads;
+        switch (size)
+        {
+            case tiny:
+                numWrites = 100;
+                numReads = 10;
+                break;
+            case small:
+                numWrites = 100_000;
+                numReads = 1_000;
+                break;
+            case medium:
+                numWrites = 1_000_000;
+                numReads = 1_000;
+                break;
+            case large:
+                numWrites = 10_000_000;
+                numReads = 1_000;
+                break;
+            case benchmark:
+                numWrites = 1_000_000;
+                numReads = numWrites * 10;
+                break;
+            default:
+                throw new AssertionError("Unknown size: " + size);
+        }
+        var minToken = 0;
+        var maxToken = (1 << 8) * numWrites;
+        int numStores = 10;
+        qt().withSeed(-1464527987857660885L).withExamples(1).check(rs -> {
+            timed("write(" + numWrites + ")", () -> writeRecords(rs, numStores, tables, minToken, maxToken, numWrites));
+            if (INCLUDE_FLUSH)
+                timed("flush(writes=" + numWrites + ")", () -> FBUtilities.waitOnFutures(Keyspace.open("system_accord").flush(ColumnFamilyStore.FlushReason.UNIT_TESTS)));
+            var warmupReads = Math.max(1, (int) (numReads * .2));
+            timed("warmup read(" + warmupReads + ")", () -> readRecords(rs, warmupReads));
+            timed("read(" + numReads + ")", () -> readRecords(rs, numReads));
+        });
+    }
+
+    private static void timed(String name, Runnable fn)
+    {
+        logger.warn("Task {} starting...", name);
+        long startNs = nanoTime();
+        try
+        {
+            fn.run();
+        }
+        catch (Throwable t)
+        {
+            logger.warn("Task {} failed after {}ms", name, TimeUnit.NANOSECONDS.toMillis(nanoTime() - startNs));
+            throw t;
+        }
+        logger.warn("Task {} completed after {}ms", name, TimeUnit.NANOSECONDS.toMillis(nanoTime() - startNs));
+    }
+
+    private static class RangeWrapper
+    {
+        final TokenRange[] ranges;
+        final IntervalTree<RoutingKey, TxnId, Interval<RoutingKey, TxnId>> tree;
+
+        RangeWrapper(TokenRange[] ranges, IntervalTree<RoutingKey, TxnId, Interval<RoutingKey, TxnId>> tree)
+        {
+            this.ranges = ranges;
+            this.tree = tree;
+        }
+    }
+
+    private Int2ObjectHashMap<Map<TableId, long[]>> store2Table2Tokens;
+    private Int2ObjectHashMap<Map<TableId, RangeWrapper>> store2Table2Ranges;
+
+    private void readRecords(RandomSource rs, int numRecords)
+    {
+        logger.warn("The bookkeeping is {} bytes", ObjectSizes.measureDeep(storeToTableToRoutingKeysToTxns));
+        // sort the tokens
+        if (domain == Routable.Domain.Key && store2Table2Tokens == null)
+        {
+            store2Table2Tokens = new Int2ObjectHashMap<>();
+            timed("Model building: key", () -> storeToTableToRoutingKeysToTxns.forEachInt((storeId, actual) -> {
+                Map<TableId, long[]> map = new HashMap<>();
+                for (var e : actual.entrySet())
+                {
+                    var keys = e.getValue().keySet();
+                    long[] tokens = new long[keys.size()];
+                    var it = keys.iterator();
+                    for (int i = 0; it.hasNext(); i++)
+                        tokens[i] = it.nextLong();
+                    Arrays.sort(tokens);
+                    map.put(e.getKey(), tokens);
+                }
+                store2Table2Tokens.put(storeId, map);
+            }));
+            store2Table2Ranges = null;
+        }
+        else if (domain == Routable.Domain.Range && store2Table2Ranges == null)
+        {
+            store2Table2Ranges = new Int2ObjectHashMap<>();
+            timed("Model building: range", () -> storeToTableToRangesToTxns.forEachInt((storeId, actual) -> {
+                Map<TableId, RangeWrapper> map = new HashMap<>();
+                for (var e : actual.entrySet())
+                {
+                    TableId tableId = e.getKey();
+                    Map<TokenRange, List<TxnId>> range2Txns = e.getValue();
+                    var keys = range2Txns.keySet();
+                    TokenRange[] ranges = new TokenRange[keys.size()];
+                    var it = keys.iterator();
+                    var builder = new IntervalTree.Builder<RoutingKey, TxnId, Interval<RoutingKey, TxnId>>();
+                    for (int i = 0; it.hasNext(); i++)
+                    {
+                        TokenRange r = ranges[i] = it.next();
+                        List<TxnId> txns = range2Txns.get(r);
+                        txns.forEach(txnId -> builder.add(new Interval<>(r.start(), r.end(), txnId)));
+                    }
+                    Arrays.sort(ranges, Range::compare);
+                    map.put(tableId, new RangeWrapper(ranges, builder.build()));
+                }
+                store2Table2Ranges.put(storeId, map);
+            }));
+            store2Table2Tokens = null;
+        }
+        long[] samples = new long[numRecords];
+        int[] counts = new int[numRecords];
+        int size = 0;
+        int numReadSizeAborts = 0;
+        try
+        {
+            for (int i = 0; i < numRecords; i++)
+            {
+                int store;
+                TableId table;
+                Set<TxnId> expected = new HashSet<>();
+                TokenKey start, end;
+                switch (domain)
+                {
+                    case Key:
+                    {
+                        store = rs.pick(storeToTableToRoutingKeysToTxns.keySet());
+                        var actual = this.storeToTableToRoutingKeysToTxns.get(store);
+                        var tableToTokens = store2Table2Tokens.get(store);
+
+                        table = rs.pick(actual.keySet());
+                        var tokens = tableToTokens.get(table);
+
+                        var offset = rs.nextInt(0, tokens.length);
+                        var endOffset = offset == tokens.length - 1 ? tokens.length - 1 : offset + rs.nextInt(1, Math.min(3, tokens.length - offset));
+                        IntStream.range(offset + 1, endOffset + 1).mapToLong(o -> tokens[o]).forEach(token -> expected.addAll(actual.get(table).get(token)));
+
+                        start = new TokenKey(table, new Murmur3Partitioner.LongToken(tokens[offset]));
+                        end = new TokenKey(table, new Murmur3Partitioner.LongToken(tokens[endOffset]));
+                    }
+                    break;
+                    case Range:
+                    {
+                        store = rs.pick(storeToTableToRangesToTxns.keySet());
+                        var tableToRangesToTxns = storeToTableToRangesToTxns.get(store);
+                        var tableToRanges = store2Table2Ranges.get(store);
+
+                        table = rs.pick(tableToRangesToTxns.keySet());
+                        var wrapper = tableToRanges.get(table);
+                        var ranges = wrapper.ranges;
+                        var tree = wrapper.tree;
+                        var range = rs.pick(ranges);
+                        var a = tokenValue(range.start());
+                        var b = tokenValue(range.end());
+                        start = new TokenKey(table, new Murmur3Partitioner.LongToken(a + 1));
+                        end = new TokenKey(table, new Murmur3Partitioner.LongToken(b - 1));
+                        expected.addAll(tree.search(start, end));
+                        assert !expected.isEmpty();
+                    }
+                    break;
+                    default:
+                        throw new IllegalArgumentException("Unknown domain: " + domain);
+                }
+
+                var startNs = nanoTime();
+                Set<TxnId> actual = read(store, start, end);
+                var durationNs = nanoTime() - startNs;
+                samples[size] = durationNs;
+                counts[size++] = actual.size();
+                if (slow(durationNs))
+                    logger.warn("Slow search: i={}, store={}, [{}, {}), results={}", i, store, start, end, actual.size());
+                if (VALIDATE)
+                {
+                    try
+                    {
+                        Assertions.assertThat(actual).describedAs("[%s, %s)", start, end).isEqualTo(expected);
+                    }
+                    catch (Throwable t)
+                    {
+                        logSamples(samples, counts, size);
+                        throw t;
+                    }
+                }
+            }
+        }
+        finally
+        {
+            logger.info("Number of aborts due to size: {}", numReadSizeAborts);
+            logSamples(samples, counts, size);
+        }
+    }
+
+    private static long tokenValue(RoutingKey start)
+    {
+        return ((AccordRoutingKey.TokenKey) start).token().getLongValue();
+    }
+
+    private static boolean slow(long durationNs)
+    {
+        return durationNs >= SLOW_NS;
+    }
+
+    private Set<TxnId> read(int store, AccordRoutingKey start, AccordRoutingKey end)
+    {
+        switch (read)
+        {
+            case INDEX:
+                return readIndex(store, start, end);
+            case CQL:
+                return readCQL(store, start, end);
+            default:
+                throw new AssertionError("Unknown read type: " + read);
+        }
+    }
+
+    private Set<TxnId> readIndex(int store, AccordRoutingKey start, AccordRoutingKey end)
+    {
+        return searcher.intersects(store, start, end);
+    }
+
+    private Set<TxnId> readCQL(int store, AccordRoutingKey start, AccordRoutingKey end)
+    {
+        Set<TxnId> actual = new HashSet<>();
+        try
+        {
+            UntypedResultSet results = execute("SELECT txn_id FROM system_accord.commands WHERE store_id = ? AND route > ? AND route <= ?", store, OrderedRouteSerializer.serializeRoutingKey(start), OrderedRouteSerializer.serializeRoutingKey(end));
+            for (var row : results)
+                actual.add(AccordKeyspace.deserializeTxnId(row));
+        }
+        catch (ReadSizeAbortException e)
+        {
+            // don't count it...
+            logger.warn("Abort query to [{}, {}) do to size", start, end);
+            return null;
+        }
+        return actual;
+    }
+
+    private void logSamples(long[] samples, int[] counts, int size)
+    {
+        if (size == 0)
+        {
+            logger.warn("No logs sampled");
+            return;
+        }
+        Arrays.sort(samples, 0, size);
+        Arrays.sort(counts, 0, size);
+        StringBuilder sb = new StringBuilder();
+        sb.append("Samples: ").append(size);
+        sb.append("\nLatenciy:");
+        sb.append("\n  Min (micro): ").append(TimeUnit.NANOSECONDS.toMicros(samples[0]));
+        sb.append("\n  Max (micro): ").append(TimeUnit.NANOSECONDS.toMicros(samples[size - 1]));
+        sb.append("\n  Median (micro): ").append(TimeUnit.NANOSECONDS.toMicros(samples[size / 2]));
+        sb.append("\n  Avg (micro): ").append(TimeUnit.NANOSECONDS.toMicros((long) LongStream.of(samples).limit(size).average().getAsDouble()));
+        sb.append("\nCounts:");
+        sb.append("\n  Min: ").append(counts[0]);
+        sb.append("\n  Max: ").append(counts[size - 1]);
+        sb.append("\n  Median: ").append(counts[size / 2]);
+        sb.append("\n  Avg: ").append((int) IntStream.of(counts).limit(size).average().getAsDouble());
+        logger.info(sb.toString());
+    }
+
+    private void writeRecords(RandomSource rs,
+                              int numStores,
+                              List<TableId> tables,
+                              int minToken, int maxToken,
+                              int numRecords)
+    {
+        var cql = "INSERT INTO system_accord.commands (store_id, domain, txn_id, status, route, durability) VALUES (?, ?, ?, ?, ?, ?)";
+        for (int i = 0; i < numRecords; i++)
+        {
+            int store = rs.nextInt(0, numStores);
+            TxnId txnId = new TxnId(0, 1000 + i, Txn.Kind.Write, domain, NODE);
+            int domain = txnId.domain().ordinal();
+            int status = SaveStatus.PreCommitted.ordinal();
+            ByteBuffer routeBB;
+            try
+            {
+                Route<?> route = createRoute(rs, numRecords, i, rs.nextInt(1, 20), tables, minToken, maxToken);
+                for (var u : route)
+                {
+                    switch (u.domain())
+                    {
+                        case Key:
+                        {
+                            AccordRoutingKey key = (AccordRoutingKey) u;
+                            var table = key.table();
+                            var token = key.token().getLongValue();
+                            storeToTableToRoutingKeysToTxns.computeIfAbsent(store, ignore -> new HashMap<>())
+                                                           .computeIfAbsent(table, ignore -> new Long2ObjectHashMap<>())
+                                                           .computeIfAbsent(token, ignore -> new ArrayList<>())
+                                                           .add(txnId);
+                        }
+                        break;
+                        case Range:
+                        {
+                            TokenRange range = (TokenRange) u;
+                            var table = range.table();
+                            storeToTableToRangesToTxns.computeIfAbsent(store, ignore -> new HashMap<>())
+                                                      .computeIfAbsent(table, ignore -> new HashMap<>())
+                                                      .computeIfAbsent(range, ignore -> new ArrayList<>())
+                                                      .add(txnId);
+                        }
+                        break;
+                        default:
+                            throw new AssertionError("Unexpected domain: " + u.domain());
+                    }
+                }
+                routeBB = AccordKeyspace.serializeRoute(route);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+            int durability = Status.Durability.NotDurable.ordinal();
+
+            execute(cql, store, domain, AccordKeyspace.serializeTimestamp(txnId), status, routeBB, durability);
+        }
+    }
+
+    private Route<?> createRoute(RandomSource rs, int numRecords, int index, int numKeys, List<TableId> tables, int minToken, int maxToken)
+    {
+        switch (domain)
+        {
+            case Key:
+            {
+                TreeSet<AccordRoutingKey> keys = new TreeSet<>();
+                while (keys.size() < numKeys)
+                {
+                    var table = rs.pick(tables);
+                    var token = new Murmur3Partitioner.LongToken(rs.nextInt(minToken, maxToken));
+                    keys.add(new TokenKey(table, token));
+                }
+                return new FullKeyRoute(keys.first(), true, keys.toArray(RoutingKey[]::new));
+            }
+            case Range:
+            {
+                TreeSet<TokenRange> ranges = new TreeSet<>(Range::compareTo);
+                RoutingKey routingKey = null;
+                var domain = maxToken - minToken + 1;
+                var delta = domain / numRecords;
+                var sub_delta = delta / numKeys;
+                while (ranges.size() < numKeys)
+                {
+                    var table = rs.pick(tables);
+                    int a, b;
+                    switch (pattern)
+                    {
+                        case RANDOM:
+                        {
+                            a = rs.nextInt(minToken, maxToken);
+                            b = rs.nextInt(minToken, maxToken);
+                            while (a == b)
+                                b = rs.nextInt(minToken, maxToken);
+                            if (a > b)
+                            {
+                                var tmp = a;
+                                a = b;
+                                b = tmp;
+                            }
+                        }
+                        break;
+                        case NO_OVERLAP:
+                        {
+                            a = delta * index + (sub_delta * ranges.size());
+                            b = a + sub_delta;
+                        }
+                        break;
+                        default:
+                            throw new IllegalArgumentException("Unknown pattern: " + pattern);
+                    }
+                    ranges.add(new TokenRange(new TokenKey(table, new Murmur3Partitioner.LongToken(a)), new TokenKey(table, new Murmur3Partitioner.LongToken(b))));
+                    if (routingKey == null)
+                    {
+                        routingKey = new TokenKey(table, new Murmur3Partitioner.LongToken(b));
+                    }
+                }
+                return Ranges.ofSorted(ranges.toArray(Range[]::new)).toRoute(routingKey);
+            }
+            default:
+                throw new IllegalArgumentException("Unknown domain: " + domain);
+        }
+    }
+}

test/unit/org/apache/cassandra/index/accord/CheckpointIntervalArrayIndexTest.java

@@ -0,0 +1,441 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.index.accord.CheckpointIntervalArrayIndex.Interval;
+import org.apache.cassandra.index.accord.IndexDescriptor.IndexComponent;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+public class CheckpointIntervalArrayIndexTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(CheckpointIntervalArrayIndexTest.class);
+
+    static
+    {
+        DatabaseDescriptor.clientInitialization();
+    }
+
+    private static final byte[] EMPTY = new byte[0];
+    private static final TreeSet<Interval> EMPTY_TREE_SET = new TreeSet<>();
+
+    private static final Gen.IntGen MAX_TOKEN_GEN = Gens.pickInt(1 << 14,
+                                                                 1 << 16,
+                                                                 1 << 20,
+                                                                 1 << 30);
+
+    private enum Pattern { RANDOM, NO_OVERLAP, PARTIAL_OVERLAP }
+
+    private static final Gen<Pattern> PATTERN_GEN = Gens.enums().all(Pattern.class);
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+    private int generation = 0;
+
+    @Test
+    public void simple() throws IOException
+    {
+        int bytesPerKey = Integer.BYTES;
+        int bytesPerValue = 0;
+
+        List<Interval> list = new ArrayList<>(10);
+        list.add(new Interval(bytes(Integer.MIN_VALUE).array(), bytes(Integer.MAX_VALUE).array(), EMPTY));
+        for (int i = 0; i < 10; i++)
+            list.add(new Interval(bytes(i).array(), bytes(i + 1).array(), EMPTY));
+
+        try (var searcher = index(bytesPerKey, bytesPerValue, list))
+        {
+            Set<List<Integer>> expected = Set.of(List.of(-2147483648, 2147483647),
+                                                 List.of(2, 3),
+                                                 List.of(3, 4));
+            Set<List<Integer>> actual = new HashSet<>();
+            var stats = searcher.intersects(bytes(2).array(), bytes(4).array(), value -> actual.add(List.of(ByteBuffer.wrap(value.start).getInt(), ByteBuffer.wrap(value.end).getInt())));
+            logger.info("Stats: {}", stats);
+            Assertions.assertThat(actual).isEqualTo(expected);
+        }
+    }
+
+    @Test
+    public void fuzzSmall()
+    {
+        var minToken = 0;
+        int numRecords = 10;
+        qt().withTimeout(Duration.ofSeconds(60)).check(rs -> fuzz(rs, minToken, MAX_TOKEN_GEN.nextInt(rs), PATTERN_GEN.next(rs), numRecords));
+    }
+
+    @Test
+    public void fuzzMedium()
+    {
+        var minToken = 0;
+        int numRecords = 1_000;
+        qt().withTimeout(Duration.ofSeconds(60)).check(rs -> fuzz(rs, minToken, MAX_TOKEN_GEN.nextInt(rs), PATTERN_GEN.next(rs), numRecords));
+    }
+
+    private void fuzz(RandomSource rs, int minToken, int maxToken, Pattern pattern, int numRecords) throws IOException
+    {
+        List<DetailedInterval> intervals = buildIntervals(rs, minToken, maxToken, pattern, numRecords);
+        List<? extends Interval> nonContainedRanges = findMissingRanges(intervals);
+
+        try (var searcher = index(Integer.BYTES, Integer.BYTES, intervals))
+        {
+            for (int i = 0, samples = rs.nextInt(Math.min(10, numRecords), Math.min(10, numRecords) * 10); i < samples; i++)
+            {
+                SearchContext ctx = rs.decide(.2) ? miss(rs, nonContainedRanges)
+                                                  : hit(rs, intervals, pattern);
+                Set<Interval> actual = new TreeSet<>();
+                try
+                {
+                    var stats = searcher.intersects(ctx.start, ctx.end, interval -> actual.add(new DetailedInterval(interval)));
+                    logger.info("[Pattern={}, size={}, expectedMatches={}, query=[{}, {})] Stats: {}", pattern, intervals.size(), ctx.expected.size(), ctx.a, ctx.b, stats);
+                }
+                catch (Throwable t)
+                {
+                    throw new AssertionError(String.format("Failure searching for [%d, %d) from %s", ctx.a, ctx.b, intervals), t);
+                }
+                Assertions.assertThat(actual).describedAs("search(%d, %d) from %s", ctx.a, ctx.b, intervals).isEqualTo(ctx.expected);
+            }
+        }
+    }
+
+    /**
+     * mutable/shared ctx to avoid allocating in a loop...
+     */
+    private SearchContext searchContext = new SearchContext();
+
+    private SearchContext miss(RandomSource rs, List<? extends Interval> nonContainedRanges)
+    {
+        var range = rs.pick(nonContainedRanges);
+        var s = unbc(range.start);
+        var e = unbc(range.end);
+        int domain = e - s;
+        int a, b;
+        if (domain == 1)
+        {
+            // you can not find multiple values within this range!
+            a = s;
+            b = e;
+        }
+        else
+        {
+            a = e == Integer.MAX_VALUE ? rs.nextInt(s, e) : rs.nextInt(s, e) + 1;
+            b = e == Integer.MAX_VALUE ? rs.nextInt(s, e) : rs.nextInt(s, e) + 1;
+            for (int i = 0; i < 42 && a == b; i++)
+                b = e == Integer.MAX_VALUE ? rs.nextInt(s, e) : rs.nextInt(s, e) + 1;
+            if (a == b)
+                throw new IllegalStateException("Unable to create missing range: " + range);
+            if (b < a)
+            {
+                var tmp = a;
+                a = b;
+                b = tmp;
+            }
+        }
+        searchContext.a = a;
+        searchContext.b = b;
+        searchContext.start = bc(a);
+        searchContext.end = bc(b);
+        searchContext.expected = EMPTY_TREE_SET;
+        return searchContext;
+    }
+
+    private SearchContext hit(RandomSource rs, List<DetailedInterval> intervals, Pattern pattern)
+    {
+        int numRecords =  intervals.size();
+        DetailedInterval first, second;
+        do
+        {
+            var offset = rs.nextInt(0, numRecords);
+            int endOffset;
+            switch (pattern)
+            {
+                case PARTIAL_OVERLAP:
+                case RANDOM:
+                    endOffset = offset;
+                    break;
+                case NO_OVERLAP:
+                    endOffset = offset == numRecords - 1 ? offset : offset + rs.nextInt(1, Math.min(3, numRecords - offset));
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unknown pattern: " + pattern);
+            }
+            first = intervals.get(offset);
+            second = intervals.get(endOffset);
+        }
+        while (first.compareTo(second) == 0 && first.size() == 1);
+        int a, b;
+        a = rs.nextInt(unbc(first.start), unbc(first.end)) + 1;
+        b = rs.nextInt(unbc(second.start), unbc(second.end)) + 1;
+        while (a == b)
+            b = rs.nextInt(unbc(second.start), unbc(second.end)) + 1;
+        if (b < a)
+        {
+            var tmp = b;
+            b = a;
+            a = tmp;
+        }
+
+        searchContext.start = bc(a);
+        searchContext.end = bc(b);
+
+        searchContext.expected = intervals.stream().filter(i -> i.intersects(searchContext.start, searchContext.end)).collect(Collectors.toCollection(TreeSet::new));
+        Assertions.assertThat(searchContext.expected).isNotEmpty();
+        return searchContext;
+    }
+
+    private static List<DetailedInterval> buildIntervals(RandomSource rs, int minToken, int maxToken, Pattern pattern, int numRecords)
+    {
+        List<DetailedInterval> intervals = new ArrayList<>(numRecords);
+        {
+            var domain = maxToken - minToken + 1;
+            var delta = domain / numRecords;
+            var sub_delta = delta / 2;
+            for (int i = 0; i < numRecords; i++)
+            {
+                switch (pattern)
+                {
+                    case RANDOM:
+                    {
+                        var start = rs.nextInt(minToken, maxToken);
+                        var remaining = maxToken - start;
+                        var end = start + (remaining == 1 ? 1 : rs.nextInt(1, remaining));
+                        intervals.add(new DetailedInterval(bc(start), bc(end), bytes(i).array()));
+                    }
+                    break;
+                    case NO_OVERLAP:
+                    {
+                        var start = delta * i;
+                        var end = start + sub_delta;
+                        intervals.add(new DetailedInterval(bc(start), bc(end), bytes(i).array()));
+                    }
+                    break;
+                    case PARTIAL_OVERLAP:
+                    {
+                        if (i > 1 && rs.decide(.2))
+                        {
+                            // overlap
+                            DetailedInterval start, end;
+                            do
+                            {
+                                int numOverlaps = rs.nextInt(1, Math.min(3, intervals.size()));
+                                int offset = rs.nextInt(0, intervals.size() - numOverlaps);
+                                start = intervals.get(offset);
+                                end = intervals.get(offset + numOverlaps);
+                            }
+                            while (start.compareStart(end) == 0 && start.size() == 1);
+                            var a = rs.nextInt(unbc(start.start), unbc(start.end)) + 1;
+                            var b = rs.nextInt(unbc(end.start), unbc(end.end)) + 1;
+                            if (a == b && end.size() == 1)
+                            {
+                                while (a == b)
+                                    a = rs.nextInt(unbc(start.start), unbc(start.end)) + 1;
+                            }
+                            else
+                            {
+                                while (a == b)
+                                    b = rs.nextInt(unbc(end.start), unbc(end.end)) + 1;
+                            }
+                            if (a > b)
+                            {
+                                var tmp = a;
+                                a = b;
+                                b = tmp;
+                            }
+                            intervals.add(new DetailedInterval(bc(a), bc(b), bytes(i).array()));
+                            intervals.sort(Comparator.naturalOrder()); // so partial can work next time
+                        }
+                        else
+                        {
+                            // no overlap
+                            var start = delta * i;
+                            var end = start + sub_delta;
+                            intervals.add(new DetailedInterval(bc(start), bc(end), bytes(i).array()));
+                        }
+                    }
+                    break;
+                    default:
+                        throw new IllegalArgumentException("Unknown pattern: " + pattern);
+                }
+            }
+            intervals.sort(Comparator.naturalOrder());
+        }
+        return intervals;
+    }
+
+    private static List<DetailedInterval> findMissingRanges(List<? extends Interval> intervals)
+    {
+        List<DetailedInterval> list = new ArrayList<>();
+        list.add(new DetailedInterval(bc(Integer.MIN_VALUE), intervals.get(0).start, bytes(0).array()));
+        // track current visable coverage
+        int end = unbc(intervals.get(0).end);
+        for (var i : intervals)
+        {
+            int istar = unbc(i.start);
+            int iend = unbc(i.end);
+            if (end >= istar)
+            {
+                // current scope includes this range
+                end = Math.max(end, iend);
+            }
+            else
+            {
+                // range doesn't intersect, and a new start/end are formed!
+                list.add(new DetailedInterval(bc(end), bc(istar), bytes(list.size()).array()));
+                end = iend;
+            }
+        }
+        list.add(new DetailedInterval(bc(end), bc(Integer.MAX_VALUE), bytes(list.size()).array()));
+        return list;
+    }
+
+    private static class DetailedInterval extends Interval
+    {
+        public DetailedInterval(byte[] start, byte[] end, byte[] value)
+        {
+            super(start, end, value);
+        }
+
+        public DetailedInterval(Interval other)
+        {
+            super(other);
+        }
+
+        public int size()
+        {
+            return unbc(end) - unbc(start);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "[" + unbc(start) + ", " + unbc(end) + ") -> " + ByteBuffer.wrap(value).getInt();
+        }
+    }
+
+    private static byte[] bc(int value)
+    {
+        ByteBuffer bb = bytes(value);
+        var bs = Int32Type.instance.asComparableBytes(ByteBufferAccessor.instance, bb, ByteComparable.Version.OSS50);
+        return ByteSourceInverse.readBytes(bs);
+    }
+
+    private static int unbc(byte[] bc)
+    {
+        return Int32Type.instance.fromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(bc)), ByteComparable.Version.OSS50).getInt();
+    }
+
+    @SuppressWarnings({ "IOResourceOpenedButNotSafelyClosed", "resource" })
+    private Searcher index(int bytesPerKey, int bytesPerValue, List<? extends Interval> sortedIntervals) throws IOException
+    {
+        IndexDescriptor descriptor = nextDescriptor();
+
+        var writer = new CheckpointIntervalArrayIndex.SegmentWriter(descriptor, bytesPerKey, bytesPerValue);
+        var metas = writer.write(sortedIntervals.toArray(Interval[]::new));
+
+        // going through the RouteIndexFormat isn't required for this test, but it helps improve coverage there...
+        Segment segment = new Segment(ImmutableMap.of(new Group(0, TableId.fromUUID(new UUID(0, 0))), new Segment.Metadata(metas, ByteArrayUtil.EMPTY_BYTE_ARRAY, ByteArrayUtil.EMPTY_BYTE_ARRAY)));
+        RouteIndexFormat.appendSegment(descriptor, segment);
+
+        Map<IndexComponent, FileHandle> files = new EnumMap<>(IndexComponent.class);
+        for (IndexComponent c : descriptor.getLiveComponents())
+            files.put(c, new FileHandle.Builder(descriptor.fileFor(c)).mmapped(true).complete());
+        List<Segment> segments = RouteIndexFormat.readSegements(files);
+        files.remove(IndexComponent.SEGMENT).close();
+        files.remove(IndexComponent.METADATA).close();
+
+        var searcher = new CheckpointIntervalArrayIndex.SegmentSearcher(files.get(IndexComponent.CINTIA_SORTED_LIST).sharedCopy(), metas.get(IndexComponent.CINTIA_SORTED_LIST).offset,
+                                                                        files.get(IndexComponent.CINTIA_CHECKPOINTS).sharedCopy(), metas.get(IndexComponent.CINTIA_CHECKPOINTS).offset);
+        return new Searcher()
+        {
+            @Override
+            public CheckpointIntervalArrayIndex.Stats intersects(byte[] start, byte[] end, Consumer<Interval> callback) throws IOException
+            {
+                return searcher.intersects(start, end, callback);
+            }
+
+            @Override
+            public void close()
+            {
+                searcher.close();
+                for (var fh : files.values())
+                    fh.close();
+            }
+        };
+    }
+
+    private IndexDescriptor nextDescriptor()
+    {
+        return IndexDescriptor.create(new Descriptor(new File(folder.getRoot()), "test", "test", new SequenceBasedSSTableId(generation++)),
+                                      Murmur3Partitioner.instance,
+                                      new ClusteringComparator());
+    }
+
+    private static class SearchContext
+    {
+        TreeSet<Interval> expected;
+        byte[] start, end;
+        int a, b;
+    }
+
+    public interface Searcher extends Closeable
+    {
+        CheckpointIntervalArrayIndex.Stats intersects(byte[] start, byte[] end, Consumer<Interval> callback) throws IOException;
+    }
+}

test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java

@@ -0,0 +1,508 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.accord;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.Iterables;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import accord.api.RoutingKey;
+import accord.local.Node;
+import accord.local.SaveStatus;
+import accord.local.Status.Durability;
+import accord.primitives.FullKeyRoute;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
+import accord.primitives.Route;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.Property.Command;
+import accord.utils.Property.Commands;
+import accord.utils.Property.UnitCommand;
+import accord.utils.RandomSource;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.Long2ObjectHashMap;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordKeyspace;
+import org.apache.cassandra.service.accord.AccordService;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.RTree;
+import org.apache.cassandra.utils.RangeTree;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.stateful;
+
+public class RouteIndexTest extends CQLTester.InMemory
+{
+    static
+    {
+        // since this test does frequent truncates, the info table gets updated and forced flushed... which is 90% of the cost of this test...
+        // this flag disables that flush
+        CassandraRelevantProperties.UNSAFE_SYSTEM.setBoolean(true);
+    }
+
+    private static final Node.Id NODE = new Node.Id(42);
+    private static final int MIN_TOKEN = 0;
+    private static final int MAX_TOKEN = 1 << 18;
+    private static final int TOKEN_RANGE_SIZE = MAX_TOKEN - MIN_TOKEN + 1;
+    private static final Gen.IntGen NUM_STORES_GEN = Gens.ints().between(1, 10);
+    private static final Gen.IntGen NUM_TABLES_GEN = Gens.ints().between(1, 10);
+    private static final Gen<Gen.IntGen> TOKEN_DISTRIBUTION = Gens.mixedDistribution(MIN_TOKEN, MAX_TOKEN + 1);
+    private static final Gen<Gen.IntGen> RANGE_SIZE_DISTRIBUTION = Gens.mixedDistribution(10, (int) (TOKEN_RANGE_SIZE * .01));
+    private static final Gen<Gen<Domain>> DOMAIN_DISTRIBUTION = Gens.mixedDistribution(Domain.values());
+    private static RoutesSearcher ROUTES_SEARCHER = null;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CQLTester.InMemory.setUpClass();
+        DatabaseDescriptor.setIncrementalBackupsEnabled(false);
+        ROUTES_SEARCHER = new RoutesSearcher();
+    }
+
+    @Test
+    public void test()
+    {
+        cfs().disableAutoCompaction(); // let the test control compaction
+        //TODO (coverage): include with the ability to mark ranges as durable for compaction cleanup
+        AccordService.unsafeSetNoop(); // disable accord service since compaction touches it.  It would be nice to include this for cleanup support....
+        stateful().withExamples(50).check(new Commands<State, ColumnFamilyStore>()
+        {
+            @Override
+            public Gen<State> genInitialState()
+            {
+                return rs -> new State(rs);
+            }
+
+            @Override
+            public ColumnFamilyStore createSut(State state)
+            {
+                return cfs();
+            }
+
+            @Override
+            public Gen<Command<State, ColumnFamilyStore, ?>> commands(State state)
+            {
+                Map<Gen<Command<State, ColumnFamilyStore, ?>>, Integer> possible = new HashMap<>();
+                possible.put(ignore -> FLUSH, 1);
+                possible.put(ignore -> COMPACT, 1);
+                possible.put(rs -> {
+                    int storeId = rs.nextInt(0, state.numStores);
+                    Domain domain = state.domainGen.next(rs);
+                    TxnId txnId = state.nextTxnId(domain);
+                    Route<?> route = createRoute(state, rs, domain, rs.nextInt(1, 20));
+                    return new InsertTxn(storeId, txnId, SaveStatus.PreAccepted, Durability.NotDurable, route);
+                }, 10);
+                possible.put(rs -> new RangeSearch(rs.nextInt(0, state.numStores), state.rangeGen.next(rs)), 1);
+                if (!state.storeToTableToRangesToTxns.isEmpty())
+                {
+                    possible.put(rs -> {
+                        int storeId = rs.pick(state.storeToTableToRangesToTxns.keySet());
+                        var tables = state.storeToTableToRangesToTxns.get(storeId);
+                        TableId tableId = rs.pick(tables.keySet());
+                        var ranges = tables.get(tableId);
+                        TreeSet<TokenRange> distinctRanges = ranges.stream().map(Map.Entry::getKey).collect(Collectors.toCollection(() -> new TreeSet<>(TokenRange::compareTo)));
+                        TokenRange range;
+                        if (distinctRanges.size() == 1)
+                        {
+                            range = Iterables.getFirst(distinctRanges, null);
+                        }
+                        else
+                        {
+                            switch (rs.nextInt(0, 2))
+                            {
+                                case 0: // perfect match
+                                    range = rs.pick(distinctRanges);
+                                    break;
+                                case 1: // mutli-match
+                                {
+                                    TokenRange a = rs.pick(distinctRanges);
+                                    TokenRange b = rs.pick(distinctRanges);
+                                    while (a.equals(b))
+                                        b = rs.pick(distinctRanges);
+                                    if (b.compareTo(a) < 0)
+                                    {
+                                        TokenRange tmp = a;
+                                        a = b;
+                                        b = tmp;
+                                    }
+                                    range = new TokenRange((AccordRoutingKey) a.start(), (AccordRoutingKey) b.end());
+                                }
+                                break;
+                                default:
+                                    throw new AssertionError();
+                            }
+                        }
+                        return new RangeSearch(storeId, range);
+                    }, 5);
+                }
+                return Gens.oneOf(possible);
+            }
+
+            @Override
+            public void destroySut(ColumnFamilyStore sut)
+            {
+                cfs().truncateBlocking();
+            }
+        });
+    }
+
+    private static ColumnFamilyStore cfs()
+    {
+        return Keyspace.open(SchemaConstants.ACCORD_KEYSPACE_NAME)
+                       .getColumnFamilyStore(AccordKeyspace.COMMANDS);
+    }
+
+    private static Gen<TokenRange> rangeGen(RandomSource rand, List<TableId> tables)
+    {
+        Gen.IntGen tokenGen = TOKEN_DISTRIBUTION.next(rand);
+        Gen<TableId> tableIdGen = Gens.mixedDistribution(tables).next(rand);
+        switch (rand.nextInt(0, 3))
+        {
+            case 0: // pure random
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int b = tokenGen.nextInt(rs);
+                    while (a == b)
+                        b = tokenGen.nextInt(rs);
+                    if (a > b)
+                    {
+                        int tmp = a;
+                        a = b;
+                        b = tmp;
+                    }
+                    TableId tableId = tableIdGen.next(rs);
+                    return new TokenRange(new TokenKey(tableId, new LongToken(a)),
+                                          new TokenKey(tableId, new LongToken(b)));
+                };
+            case 1: // small range
+                Gen.IntGen rangeSizeGen = RANGE_SIZE_DISTRIBUTION.next(rand);
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int rangeSize = rangeSizeGen.nextInt(rs);
+                    int b = a + rangeSize;
+                    if (b > MAX_TOKEN)
+                    {
+                        b = a;
+                        a = b - rangeSize;
+                    }
+                    TableId tableId = tableIdGen.next(rs);
+                    return new TokenRange(new TokenKey(tableId, new LongToken(a)),
+                                          new TokenKey(tableId, new LongToken(b)));
+                };
+            case 2: // single element
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int b = a + 1;
+                    TableId tableId = tableIdGen.next(rs);
+                    return new TokenRange(new TokenKey(tableId, new LongToken(a)),
+                                          new TokenKey(tableId, new LongToken(b)));
+                };
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private static Route<?> createRoute(State state, RandomSource rs, Domain domain, int numKeys)
+    {
+        switch (domain)
+        {
+            case Key:
+            {
+                TreeSet<AccordRoutingKey> keys = new TreeSet<>();
+                while (keys.size() < numKeys)
+                {
+                    var table = rs.pick(state.tables);
+                    var token = new LongToken(state.tokenGen.nextInt(rs));
+                    keys.add(new TokenKey(table, token));
+                }
+                return new FullKeyRoute(keys.first(), true, keys.toArray(RoutingKey[]::new));
+            }
+            case Range:
+            {
+                TreeSet<TokenRange> set = new TreeSet<>(Range::compareTo);
+                while (set.size() < numKeys)
+                    set.add(state.rangeGen.next(rs));
+                return Ranges.ofSorted(set.toArray(Range[]::new)).toRoute(set.first().end());
+            }
+            default:
+                throw new IllegalArgumentException("Unknown domain: " + domain);
+        }
+    }
+
+    private class InsertTxn implements UnitCommand<State, ColumnFamilyStore>
+    {
+        private static final String cql = "INSERT INTO system_accord.commands (store_id, domain, txn_id, status, route, durability) VALUES (?, ?, ?, ?, ?, ?)";
+        private final int storeId;
+        private final TxnId txnId;
+        private final SaveStatus saveStatus;
+        private final Durability durability;
+        private final Route<?> route;
+
+        private InsertTxn(int storeId, TxnId txnId, SaveStatus saveStatus, Durability durability, Route<?> route)
+        {
+            this.storeId = storeId;
+            this.txnId = txnId;
+            this.saveStatus = saveStatus;
+            this.durability = durability;
+            this.route = route;
+        }
+
+        @Override
+        public void applyUnit(State state)
+        {
+            for (var u : route)
+            {
+                switch (u.domain())
+                {
+                    case Key:
+                    {
+                        AccordRoutingKey key = (AccordRoutingKey) u;
+                        var table = key.table();
+                        var token = key.token().getLongValue();
+                        state.storeToTableToRoutingKeysToTxns.computeIfAbsent(storeId, ignore -> new HashMap<>())
+                                                             .computeIfAbsent(table, ignore -> new Long2ObjectHashMap<>())
+                                                             .computeIfAbsent(token, ignore -> new ArrayList<>())
+                                                             .add(txnId);
+                    }
+                    break;
+                    case Range:
+                    {
+                        TokenRange range = (TokenRange) u;
+                        var table = range.table();
+                        state.storeToTableToRangesToTxns.computeIfAbsent(storeId, ignore -> new HashMap<>())
+                                                        .computeIfAbsent(table, ignore -> rangeTree())
+                                                        .add(range, txnId);
+                    }
+                    break;
+                    default:
+                        throw new AssertionError("Unexpected domain: " + u.domain());
+                }
+            }
+        }
+
+        @Override
+        public void runUnit(ColumnFamilyStore sut) throws Throwable
+        {
+            execute(cql, storeId, txnId.domain().ordinal(), AccordKeyspace.serializeTimestamp(txnId), saveStatus.ordinal(), AccordKeyspace.serializeRoute(route), durability.ordinal());
+        }
+
+        @Override
+        public String toString()
+        {
+            return "InsertTxn{" +
+                   "storeId=" + storeId +
+                   ", txnId=" + txnId +
+                   ", saveStatus=" + saveStatus +
+                   ", durability=" + durability +
+                   ", route=" + route +
+                   '}';
+        }
+    }
+
+    private class RangeSearch implements Command<State, ColumnFamilyStore, Set<TxnId>>
+    {
+        private final int storeId;
+        private final TokenRange range;
+
+        private RangeSearch(int storeId, TokenRange range)
+        {
+            this.storeId = storeId;
+            this.range = range;
+        }
+
+        @Override
+        public Set<TxnId> apply(State state) throws Throwable
+        {
+            var tables = state.storeToTableToRangesToTxns.get(storeId);
+            if (tables == null) return Collections.emptySet();
+            var ranges = tables.get(range.table());
+            if (ranges == null) return Collections.emptySet();
+            Set<TxnId> matches = new HashSet<>();
+            ranges.search(range, e -> matches.add(e.getValue()));
+            return matches;
+        }
+
+        @Override
+        public Set<TxnId> run(ColumnFamilyStore sut) throws Throwable
+        {
+            return ROUTES_SEARCHER.intersects(storeId, range);
+        }
+
+        @Override
+        public void checkPostconditions(State state, Set<TxnId> expected,
+                                        ColumnFamilyStore sut, Set<TxnId> actual)
+        {
+            Assertions.assertThat(actual).describedAs("Unexpected txns for range %s", range).isEqualTo(expected);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "RangeSearch{" +
+                   "storeId=" + storeId +
+                   ", range=" + range +
+                   '}';
+        }
+    }
+
+    private static abstract class CassandraCommand implements UnitCommand<State, ColumnFamilyStore>
+    {
+        private final String name;
+
+        protected CassandraCommand(String name)
+        {
+            this.name = name;
+        }
+
+        @Override
+        public void applyUnit(State state)
+        {
+            // no-op
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return name;
+        }
+    }
+
+    private static final CassandraCommand FLUSH = new CassandraCommand("Flush")
+    {
+        @Override
+        public void runUnit(ColumnFamilyStore sut)
+        {
+            sut.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+        }
+    };
+
+    private static final CassandraCommand COMPACT = new CassandraCommand("Compact")
+    {
+        @Override
+        public void runUnit(ColumnFamilyStore sut)
+        {
+            try
+            {
+                sut.enableAutoCompaction();
+                FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(sut));
+            }
+            finally
+            {
+                sut.disableAutoCompaction();
+            }
+        }
+    };
+
+    private static class State
+    {
+        private final Int2ObjectHashMap<Map<TableId, Long2ObjectHashMap<List<TxnId>>>> storeToTableToRoutingKeysToTxns = new Int2ObjectHashMap<>();
+        private final Int2ObjectHashMap<Map<TableId, RangeTree<AccordRoutingKey, TokenRange, TxnId>>> storeToTableToRangesToTxns = new Int2ObjectHashMap<>();
+
+        private final int numStores;
+        private final List<TableId> tables;
+        private final Gen.IntGen tokenGen;
+        private final Gen<TokenRange> rangeGen;
+        private final Gen<Domain> domainGen;
+        private int hlc = 1000;
+
+        public State(RandomSource rs)
+        {
+            numStores = NUM_STORES_GEN.nextInt(rs);
+            tables = IntStream.range(0, NUM_TABLES_GEN.nextInt(rs))
+                              .mapToObj(i -> TableId.fromUUID(new UUID(0, i)))
+                              .collect(Collectors.toList());
+            tokenGen = TOKEN_DISTRIBUTION.next(rs);
+            rangeGen = rangeGen(rs, tables);
+            domainGen = DOMAIN_DISTRIBUTION.next(rs);
+        }
+
+        TxnId nextTxnId(Domain domain)
+        {
+            return new TxnId(1, hlc++, Txn.Kind.Write, domain, NODE);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "State{" +
+                   "numStores=" + numStores +
+                   ", tables=" + tables +
+                   '}';
+        }
+    }
+
+    private static RangeTree<AccordRoutingKey, TokenRange, TxnId> rangeTree()
+    {
+        return RTree.create(ACCESSOR);
+    }
+
+    private static final RangeTree.Accessor<AccordRoutingKey, TokenRange> ACCESSOR = new RangeTree.Accessor<AccordRoutingKey, TokenRange>()
+    {
+        @Override
+        public AccordRoutingKey start(TokenRange tokenRange)
+        {
+            return (AccordRoutingKey) tokenRange.start();
+        }
+
+        @Override
+        public AccordRoutingKey end(TokenRange tokenRange)
+        {
+            return (AccordRoutingKey) tokenRange.end();
+        }
+
+        @Override
+        public boolean contains(AccordRoutingKey start, AccordRoutingKey end, AccordRoutingKey accordRoutingKey)
+        {
+            return new TokenRange(start, end).contains(accordRoutingKey);
+        }
+
+        @Override
+        public boolean intersects(TokenRange tokenRange, AccordRoutingKey start, AccordRoutingKey end)
+        {
+            return tokenRange.compareIntersecting(new TokenRange(start, end)) == 0;
+        }
+    };
+}

test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java

@@ -569,25 +569,25 @@ public class CassandraIndexTest extends CQLTester
         Awaitility.await()
                   .atMost(1, TimeUnit.MINUTES)
                   .pollDelay(1, TimeUnit.SECONDS)
-                  .untilAsserted(() -> assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null)));
+                  .untilAsserted(() -> assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null), row("system_accord", "route", null)));
 
         String indexName = "build_remove_test_idx";
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         createIndex(String.format("CREATE INDEX %s ON %%s(c)", indexName));
 
         // check that there are no other rows in the built indexes table
-        assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null));
+        assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null), row("system_accord", "route", null));
 
         // rebuild the index and verify the built status table
         getCurrentColumnFamilyStore().rebuildSecondaryIndex(indexName);
         waitForIndexQueryable(indexName);
 
         // check that there are no other rows in the built indexes table
-        assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null));
+        assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null), row("system_accord", "route", null));
 
         // check that dropping the index removes it from the built indexes table
         dropIndex("DROP INDEX %s." + indexName);
-        assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null));
+        assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null), row("system_accord", "route", null));
     }
 
 

test/unit/org/apache/cassandra/io/util/ChecksumedDataTest.java

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+import org.junit.Test;
+
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.Property.Command;
+import accord.utils.Property.Commands;
+import accord.utils.Property.UnitCommand;
+import org.apache.cassandra.utils.FailingConsumer;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.stateful;
+
+public class ChecksumedDataTest
+{
+    public static final Supplier<Checksum> CHECKSUM_SUPPLIER = CRC32C::new;
+
+    @Test
+    public void singleType()
+    {
+        DataOutputBuffer out = new DataOutputBuffer();
+        stateful().check(new Commands<DataOutputBuffer, DataOutputBuffer>()
+        {
+            @Override
+            public Gen<DataOutputBuffer> genInitialState()
+            {
+                return ignore -> {
+                    out.clear();
+                    return out;
+                };
+            }
+
+            @Override
+            public DataOutputBuffer createSut(DataOutputBuffer dataOutputBuffer)
+            {
+                return dataOutputBuffer;
+            }
+
+            @Override
+            public Gen<Command<DataOutputBuffer, DataOutputBuffer, ?>> commands(DataOutputBuffer dataOutputBuffer)
+            {
+                return Gens.oneOf(
+                rs -> {
+                    boolean b = rs.nextBoolean();
+                    return new StatelessChecksumCommand<>(out -> out.writeBoolean(b), DataInputPlus::readBoolean, () -> b);
+                },
+                rs -> {
+                    short s = (short) rs.nextInt(Short.MIN_VALUE, Short.MAX_VALUE);
+                    return new StatelessChecksumCommand<>(out -> out.writeShort(s), DataInputPlus::readShort, () -> s);
+                },
+                rs -> {
+                    char c = (char) rs.nextInt(Character.MIN_VALUE, Character.MAX_VALUE);
+                    return new StatelessChecksumCommand<>(out -> out.writeChar(c), DataInputPlus::readChar, () -> c);
+                },
+                rs -> {
+                    int value = rs.nextInt();
+                    return new StatelessChecksumCommand<>(out -> out.writeInt(value), DataInputPlus::readInt, () -> value);
+                },
+                rs -> {
+                    float value = rs.nextFloat();
+                    return new StatelessChecksumCommand<>(out -> out.writeFloat(value), DataInputPlus::readFloat, () -> value);
+                },
+                rs -> {
+                    double value = rs.nextDouble();
+                    return new StatelessChecksumCommand<>(out -> out.writeDouble(value), DataInputPlus::readDouble, () -> value);
+                }
+                );
+            }
+        });
+    }
+
+    @Test
+    public void withState()
+    {
+        DataOutputBuffer out = new DataOutputBuffer();
+        ChecksumedDataOutputPlus checksummedOut = new ChecksumedDataOutputPlus(out, CHECKSUM_SUPPLIER);
+        checksummedOut.resetChecksum();
+        stateful().check(new Commands<ChecksumedDataOutputPlus, List<StatefulChecksumCommand<?>>>() {
+            @Override
+            public Gen<ChecksumedDataOutputPlus> genInitialState()
+            {
+                return ignore -> {
+                    out.clear();
+                    checksummedOut.resetChecksum();
+                    return checksummedOut;
+                };
+            }
+
+            @Override
+            public List<StatefulChecksumCommand<?>> createSut(ChecksumedDataOutputPlus checksumedDataOutputPlus)
+            {
+                return new ArrayList<>(1000);
+            }
+
+            @Override
+            public Gen<Command<ChecksumedDataOutputPlus, List<StatefulChecksumCommand<?>>, ?>> commands(ChecksumedDataOutputPlus checksumedDataOutputPlus)
+            {
+                return Gens.oneOf(
+                rs -> {
+                    boolean b = rs.nextBoolean();
+                    return new StatefulChecksumCommand<>(out -> out.writeBoolean(b), DataInputPlus::readBoolean, () -> b);
+                },
+                rs -> {
+                    short s = (short) rs.nextInt(Short.MIN_VALUE, Short.MAX_VALUE);
+                    return new StatefulChecksumCommand<>(out -> out.writeShort(s), DataInputPlus::readShort, () -> s);
+                },
+                rs -> {
+                    char c = (char) rs.nextInt(Character.MIN_VALUE, Character.MAX_VALUE);
+                    return new StatefulChecksumCommand<>(out -> out.writeChar(c), DataInputPlus::readChar, () -> c);
+                },
+                rs -> {
+                    int value = rs.nextInt();
+                    return new StatefulChecksumCommand<>(out -> out.writeInt(value), DataInputPlus::readInt, () -> value);
+                },
+                rs -> {
+                    float value = rs.nextFloat();
+                    return new StatefulChecksumCommand<>(out -> out.writeFloat(value), DataInputPlus::readFloat, () -> value);
+                },
+                rs -> {
+                    double value = rs.nextDouble();
+                    return new StatefulChecksumCommand<>(out -> out.writeDouble(value), DataInputPlus::readDouble, () -> value);
+                }
+                );
+            }
+
+            @Override
+            public void destroySut(List<StatefulChecksumCommand<?>> sut) throws Throwable
+            {
+                ChecksumedDataInputPlus in = new ChecksumedDataInputPlus(new DataInputBuffer(out.unsafeGetBufferAndFlip(), false), CHECKSUM_SUPPLIER);
+                for (StatefulChecksumCommand<?> cmd : sut)
+                {
+                    Assertions.assertThat(cmd.read.apply(in)).isEqualTo(cmd.expected.get());
+                    Assertions.assertThat(in.checksum().getValue()).isEqualTo(cmd.checksum);
+                }
+            }
+        });
+    }
+
+    public interface FailingFunction<I, O>
+    {
+        O apply(I input) throws Throwable;
+    }
+
+    private static class StatefulChecksumCommand<T> implements UnitCommand<ChecksumedDataOutputPlus, List<StatefulChecksumCommand<?>>>
+    {
+        private final FailingConsumer<DataOutputPlus> update;
+        private final FailingFunction<DataInputPlus, T> read;
+        private final Supplier<T> expected;
+        private Long checksum = null;
+
+        private StatefulChecksumCommand(FailingConsumer<DataOutputPlus> update, FailingFunction<DataInputPlus, T> read, Supplier<T> expected)
+        {
+            this.update = update;
+            this.read = read;
+            this.expected = expected;
+        }
+
+        @Override
+        public void applyUnit(ChecksumedDataOutputPlus out) throws Throwable
+        {
+            update.doAccept(out);
+            checksum = out.checksum().getValue();
+        }
+
+        @Override
+        public void runUnit(List<StatefulChecksumCommand<?>> sut)
+        {
+            sut.add(this);
+        }
+    }
+
+    private static class StatelessChecksumCommand<T> implements Command<DataOutputBuffer, DataOutputBuffer, Long>
+    {
+        private final FailingConsumer<DataOutputPlus> update;
+        private final FailingFunction<DataInputPlus, T> read;
+        private final Supplier<T> expected;
+
+        private StatelessChecksumCommand(FailingConsumer<DataOutputPlus> update,
+                                         FailingFunction<DataInputPlus, T> read,
+                                         Supplier<T> expected)
+        {
+            this.update = update;
+            this.read = read;
+            this.expected = expected;
+        }
+
+        @Override
+        public Long apply(DataOutputBuffer out) throws Throwable
+        {
+            out.clear();
+            ChecksumedDataOutputPlus c = new ChecksumedDataOutputPlus(out, CHECKSUM_SUPPLIER);
+            update.doAccept(c);
+            return c.checksum().getValue();
+        }
+
+        @Override
+        public Long run(DataOutputBuffer out) throws Throwable
+        {
+            out.clear();
+            update.doAccept(out);
+            ChecksumedDataInputPlus i = new ChecksumedDataInputPlus(new DataInputBuffer(out.unsafeGetBufferAndFlip(), false), CHECKSUM_SUPPLIER);
+            Assertions.assertThat(read.apply(i)).isEqualTo(expected.get());
+            return i.checksum().getValue();
+        }
+
+        @Override
+        public void checkPostconditions(DataOutputBuffer dataOutputBuffer, Long expected,
+                                        DataOutputBuffer sut, Long actual)
+        {
+            Assertions.assertThat(actual).isEqualTo(expected);
+        }
+
+        @Override
+        public String detailed(DataOutputBuffer dataOutputBuffer)
+        {
+            return expected.get().getClass().getSimpleName();
+        }
+    }
+}

test/unit/org/apache/cassandra/repair/FuzzTestBase.java

@@ -99,7 +99,6 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.LocalStrategy;
 import org.apache.cassandra.locator.Locator;
 import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.net.ConnectionType;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
@@ -117,7 +116,6 @@ import org.apache.cassandra.repair.state.SessionState;
 import org.apache.cassandra.repair.state.ValidationState;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.SystemDistributedKeyspace;
 import org.apache.cassandra.schema.TableId;
@@ -125,7 +123,6 @@ import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.Tables;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordConfigurationService;
 import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupComplete;
 import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupHistory;
@@ -141,21 +138,12 @@ import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.streaming.StreamingChannel;
 import org.apache.cassandra.streaming.StreamingDataInputPlus;
 import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.ClusterMetadataService;
-import org.apache.cassandra.tcm.Epoch;
-import org.apache.cassandra.tcm.listeners.ChangeListener;
-import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.Location;
-import org.apache.cassandra.tcm.membership.NodeAddresses;
-import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tools.nodetool.Repair;
 import org.apache.cassandra.utils.AbstractTypeGenerators;
 import org.apache.cassandra.utils.CassandraGenerators;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.Closeable;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.FailingBiConsumer;
 import org.apache.cassandra.utils.Generators;
 import org.apache.cassandra.utils.MBeanWrapper;
@@ -587,29 +575,11 @@ public abstract class FuzzTestBase extends CQLTester.InMemory
     {
         RepairType type = repairTypeGen.next(rs);
         PreviewType previewType = previewTypeGen.next(rs);
-        // TODO (required - IR) add this back and expand as part of IR integration
-//        boolean accordRepair = type == RepairType.FULL && previewType == PreviewType.NONE ? rs.nextBoolean() : false;
-        boolean accordRepair = false;
         List<String> args = new ArrayList<>();
         args.add(ks);
         List<String> tables = tablesGen.next(rs);
         args.addAll(tables);
-        if (accordRepair)
-        {
-            List<Range<Token>> ranges = new ArrayList<>(StorageService.instance.getReplicas(ks, coordinator.broadcastAddressAndPort()).ranges());
-            ranges.sort(Comparator.naturalOrder());
-            Range<Token> range = ranges.get(rs.nextInt(0, ranges.size()));
-            args.add("--start-token");
-            args.add(range.left.toString());
-            args.add("--end-token");
-            Murmur3Partitioner.LongToken left = (Murmur3Partitioner.LongToken) range.left;
-            Token right = rs.nextBoolean() ? new Murmur3Partitioner.LongToken(left.token + 100) : range.right;
-            args.add(right.toString());
-        }
-        else
-        {
-            args.add("-pr");
-        }
+        args.add("-pr");
         switch (type)
         {
             case IR:
@@ -651,8 +621,6 @@ public abstract class FuzzTestBase extends CQLTester.InMemory
         }
         if (rs.nextBoolean()) args.add("--optimise-streams");
         RepairOption options = RepairOption.parse(Repair.parseOptionMap(() -> "test", args), DatabaseDescriptor.getPartitioner());
-        if (accordRepair)
-            options = options.withAccordRepair(true);
         if (options.getRanges().isEmpty())
         {
             if (options.isPrimaryRange())
@@ -811,69 +779,11 @@ public abstract class FuzzTestBase extends CQLTester.InMemory
             }
             List<InetAddressAndPort> addresses = new ArrayList<>(nodes.keySet());
             addresses.sort(Comparator.naturalOrder());
-            NodeId tcmid = ClusterMetadata.current().directory.peerId(addresses.get(rs.nextInt(0, addresses.size())));
-            ServerTestUtils.recreateAccord(tcmid);
-            interceptTCMNotifications(tcmid);
+            AccordService.unsafeSetNoop();
 
             setupSchema();
         }
 
-        private void interceptTCMNotifications(NodeId tcmid)
-        {
-            AccordService as = (AccordService) AccordService.instance();
-            AccordConfigurationService config = as.configurationService();
-            ClusterMetadataService.instance().log().removeListener(config);
-            ClusterMetadataService.instance().log().addListener(new ChangeListener()
-            {
-                @Override
-                public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
-                {
-                    config.notifyPostCommit(sanitize(prev, tcmid), sanitize(next, tcmid), fromSnapshot);
-                }
-            });
-        }
-
-        private ClusterMetadata sanitize(ClusterMetadata metadata, NodeId tcmid)
-        {
-            if (metadata.directory.isEmpty())
-                return metadata;
-            ClusterMetadata sanitized = metadata.withDirectory(sanitize(metadata.directory, tcmid))
-                                                .withPlacements(sanitize(metadata.placements, FBUtilities.getBroadcastAddressAndPort()));
-            return sanitized;
-        }
-
-        private Directory sanitize(Directory directory, NodeId tcmid)
-        {
-            if (directory.getNodeAddresses(tcmid) == null)
-                throw new AssertionError("Expected node " + tcmid + " but not found in " + directory);
-            for (NodeId peer : directory.peerIds())
-            {
-                if (peer.equals(tcmid))
-                    continue;
-                directory = directory.without(peer);
-            }
-            directory = directory.withNodeAddresses(tcmid, NodeAddresses.current());
-            return directory;
-        }
-
-        private DataPlacements sanitize(DataPlacements placements, InetAddressAndPort endpoint)
-        {
-            DataPlacements.Builder builder = DataPlacements.builder(placements.size());
-            for (Map.Entry<ReplicationParams, DataPlacement> e : placements)
-                builder.with(e.getKey(), sanitize(placements.lastModified(), e.getValue(), endpoint));
-            return builder.build();
-        }
-
-        private DataPlacement sanitize(Epoch epoch, DataPlacement value, InetAddressAndPort endpoint)
-        {
-            DataPlacement.Builder builder = DataPlacement.builder();
-            for (Range<Token> e : value.writes.ranges())
-                builder.withWriteReplica(epoch, new Replica(endpoint, e, true));
-            for (Range<Token> e : value.reads.ranges())
-                builder.withReadReplica(epoch, new Replica(endpoint, e, true));
-            return builder.build();
-        }
-
         public Closeable addListener(MessageListener listener)
         {
             listeners.add(listener);

test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java

@@ -148,7 +148,7 @@ public class AccordCommandStoreTest
             Apply.SerializationSupport.create(txnId,
                                               route.slice(Ranges.of(TokenRange.fullRange(tableId))),
                                               1L,
-                                              Apply.Kind.Minimal,
+                                              Apply.Kind.Maximal,
                                               depTxn.keys(),
                                               executeAt,
                                               dependencies,

test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java

@@ -18,8 +18,16 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 
 import org.junit.Test;
 
@@ -43,18 +51,47 @@ import accord.primitives.Routable;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.Observable;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.schema.MemtableParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaProvider;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.utils.CassandraGenerators;
 import org.assertj.core.api.Assertions;
+import org.mockito.Mockito;
+import org.mockito.stubbing.Answer;
 
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.config.DatabaseDescriptor.setSelectedSSTableFormat;
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.setMemtable;
+import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
 import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
-import static org.apache.cassandra.service.accord.AccordTestUtils.wrapInTxn;
+import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
+import static org.apache.cassandra.utils.AccordGenerators.fromQT;
 
 public class AccordKeyspaceTest extends CQLTester.InMemory
 {
+    static
+    {
+        // since this test does frequent truncates, the info table gets updated and forced flushed... which is 90% of the cost of this test...
+        // this flag disables that flush
+        CassandraRelevantProperties.UNSAFE_SYSTEM.setBoolean(true);
+    }
+
     @Test
     public void serde()
     {
@@ -88,7 +125,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
         AccordSafeCommand safeCommand = new AccordSafeCommand(AccordTestUtils.loaded(id, null));
         safeCommand.set(committed);
 
-        Commit commit = Commit.SerializerSupport.create(id, route.slice(scope), 1, Commit.Kind.StableFastPath, Ballot.ZERO, id, partialTxn.keys(), partialTxn, partialDeps, route, null);
+        Commit commit = Commit.SerializerSupport.create(id, route.slice(scope), 1, Commit.Kind.CommitSlowPath, Ballot.ZERO, id, partialTxn.keys(), partialTxn, partialDeps, route, null);
         store.appendToJournal(commit);
 
         Mutation mutation = AccordKeyspace.getCommandMutation(store, safeCommand, 42);
@@ -97,4 +134,157 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
         Command loaded = AccordKeyspace.loadCommand(store, id);
         Assertions.assertThat(loaded).isEqualTo(committed);
     }
+
+    @Test
+    public void findOverlappingKeys()
+    {
+        var tableIdGen = fromQT(CassandraGenerators.TABLE_ID_GEN);
+        var partitionGen = fromQT(CassandraGenerators.partitioners());
+
+        var sstableFormats = DatabaseDescriptor.getSSTableFormats();
+        List<String> sstableFormatNames = new ArrayList<>(sstableFormats.keySet());
+        sstableFormatNames.sort(Comparator.naturalOrder());
+
+        List<String> memtableFormats = MemtableParams.knownDefinitions().stream()
+                                                     .filter(name -> !name.startsWith("test_") && !name.equals("default"))
+                                                     .sorted()
+                                                     .collect(Collectors.toList());
+
+        qt().check(rs -> {
+            AccordKeyspace.unsafeClear();
+            // control SSTable format
+            setSelectedSSTableFormat(sstableFormats.get(rs.pick(sstableFormatNames)));
+            // control memtable format
+            setMemtable(ACCORD_KEYSPACE_NAME, "commands_for_key", rs.pick(memtableFormats));
+
+            // define the tables w/ partitioners for the test
+            // this uses the ability to override the SchemaProvider for the keyspace and only defines the single API call expected: getTablePartitioner
+            TreeMap<TableId, IPartitioner> tables = new TreeMap<>();
+            int numTables = rs.nextInt(1, 3);
+            for (int i = 0; i < numTables; i++)
+            {
+                var tableId = tableIdGen.next(rs);
+                while (tables.containsKey(tableId))
+                    tableId = tableIdGen.next(rs);
+                tables.put(tableId, partitionGen.next(rs));
+            }
+            SchemaProvider schema = Mockito.mock(SchemaProvider.class);
+            Mockito.when(schema.getTablePartitioner(Mockito.any())).thenAnswer((Answer<IPartitioner>) invocationOnMock -> tables.get(invocationOnMock.getArgument(0)));
+            AccordKeyspace.unsafeSetSchema(schema);
+
+            int numStores = rs.nextInt(1, 3);
+
+            // The model of the DB
+            TreeMap<Integer, SortedSet<PartitionKey>> storesToKeys = new TreeMap<>();
+            // write to the table and the model
+            for (int i = 0, numKeys = rs.nextInt(10, 20); i < numKeys; i++)
+            {
+                int store = rs.nextInt(0, numStores);
+                var keys = storesToKeys.computeIfAbsent(store, ignore -> new TreeSet<>());
+                PartitionKey pk = null;
+                // LocalPartitioner may have a type with a very small domain (boolean, vector<boolean, 1>, etc.), so need to bound the attempts
+                // else this will loop forever...
+                for (int attempt = 0; attempt < 10; attempt++)
+                {
+                    TableId tableId = rs.pick(tables.keySet());
+                    IPartitioner partitioner = tables.get(tableId);
+                    ByteBuffer data = !(partitioner instanceof LocalPartitioner) ? Int32Type.instance.decompose(rs.nextInt())
+                                                                                 : fromQT(getTypeSupport(partitioner.getTokenValidator()).bytesGen()).next(rs);
+                    PartitionKey key = new PartitionKey(tableId, tables.get(tableId).decorateKey(data));
+                    if (keys.add(key))
+                    {
+                        pk = key;
+                        break;
+                    }
+                }
+                if (pk != null)
+                {
+                    try
+                    {
+                        // using Mutation directly (what we do in Accord) can break when user data is too large; leading to data loss
+                        // The memtable will allow the write, but it will be dropped when writing to the SSTable...
+                        //TODO (now, correctness): since we store the user token + user key, if a key is close to the PK limits then we could tip over and loose our CFK
+//                        new Mutation(AccordKeyspace.getCommandsForKeyPartitionUpdate(store, pk, 42, ByteBufferUtil.EMPTY_BYTE_BUFFER)).apply();
+                        execute("INSERT INTO system_accord.commands_for_key (store_id, key_token, key) VALUES (?, ?, ?)",
+                                store, AccordKeyspace.serializeRoutingKey(pk.toUnseekable()), AccordKeyspace.serializeKey(pk));
+                    }
+                    catch (IllegalArgumentException | InvalidRequestException e)
+                    {
+                        // Sometimes the types are too large (LocalPartitioner) so the mutation gets rejected... just ignore those cases
+                        // Length 69912 > max length 65535
+                        String msg = e.getMessage();
+                        if (msg != null)
+                        {
+                            if ((msg.startsWith("Length ") && msg.endsWith("> max length 65535")) // Clustering was rejected
+                                || (msg.startsWith("Key length of ") && msg.endsWith(" is longer than maximum of 65535"))) // Partition was rejected
+                            {
+                                // failed to add
+                                keys.remove(pk);
+                                continue;
+                            }
+                        }
+                        throw e;
+                    }
+                }
+            }
+
+            // read from the table and validate it matches the model
+            for (int read = 0; read < 2; read++) // read=0 is memtable, read=1 is sstable
+            {
+                {
+                    // Make sure no data was lost
+                    // An issue was found that system mutations bypass checks so make their way to the Memtable, but when we flush to SSTable
+                    // they get filtered out, causing data loss... This check is here to make sure that the data is present (test covers Memtable + SStable)
+                    // in the storage before checking if the filtering logic is correct
+                    TreeMap<Integer, SortedSet<ByteBuffer>> expectedCqlStoresToKeys = new TreeMap<>();
+                    for (var e : storesToKeys.entrySet())
+                    {
+                        int store = e.getKey();
+                        expectedCqlStoresToKeys.put(store, new TreeSet<>(e.getValue().stream().map(p -> AccordKeyspace.serializeRoutingKey(p.toUnseekable())).collect(Collectors.toList())));
+                    }
+
+                    // make sure no data loss... when this test was written sstable had all the rows but the sstable didn't... this
+                    // is mostly a santity check to detect that case early
+                    var resultSet = execute("SELECT store_id, key_token FROM system_accord.commands_for_key ALLOW FILTERING");
+                    TreeMap<Integer, SortedSet<ByteBuffer>> cqlStoresToKeys = new TreeMap<>();
+                    for (var row : resultSet)
+                    {
+                        int storeId = row.getInt("store_id");
+                        ByteBuffer bb = row.getBytes("key_token");
+                        cqlStoresToKeys.computeIfAbsent(storeId, ignore -> new TreeSet<>()).add(bb);
+                    }
+                    Assertions.assertThat(cqlStoresToKeys).isEqualTo(expectedCqlStoresToKeys);
+                }
+
+                for (int i = 0, queries = rs.nextInt(1, 5); i < queries; i++)
+                {
+                    int store = rs.pick(storesToKeys.keySet());
+                    var keysForStore = new ArrayList<>(storesToKeys.get(store));
+
+                    int offset;
+                    int offsetEnd;
+                    if (keysForStore.size() == 1)
+                    {
+                        offset = 0;
+                        offsetEnd = 1;
+                    }
+                    else
+                    {
+                        offset = rs.nextInt(0, keysForStore.size());
+                        offsetEnd = rs.nextInt(offset, keysForStore.size()) + 1;
+                    }
+                    List<PartitionKey> expected = keysForStore.subList(offset, offsetEnd);
+                    PartitionKey start = expected.get(0);
+                    PartitionKey end = expected.get(expected.size() - 1);
+
+                    AsyncChain<List<PartitionKey>> map = Observable.asChain(callback -> AccordKeyspace.findAllKeysBetween(store, start.toUnseekable(), true, end.toUnseekable(), true, callback));
+                    List<PartitionKey> actual = AsyncChains.getUnchecked(map);
+                    Assertions.assertThat(actual).isEqualTo(expected);
+                }
+
+                if (read == 0)
+                    Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStore("commands_for_key").forceBlockingFlush(UNIT_TESTS);
+            }
+        });
+    }
 }

test/unit/org/apache/cassandra/service/accord/AccordReadRepairTest.java

@@ -38,7 +38,7 @@ import static org.apache.cassandra.distributed.util.QueryResultUtil.assertThat;
 
 public class AccordReadRepairTest extends AccordTestBase
 {
-    private static final Logger logger = LoggerFactory.getLogger(org.apache.cassandra.distributed.test.accord.AccordCQLTest.class);
+    private static final Logger logger = LoggerFactory.getLogger(AccordReadRepairTest.class);
 
     @Override
     protected Logger logger()

test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java

@@ -99,6 +99,7 @@ import static java.lang.String.format;
 
 public class AccordTestUtils
 {
+    private static final AccordAgent AGENT = new AccordAgent();
     public static final TableId TABLE_ID1 = TableId.fromString("00000000-0000-0000-0000-000000000001");
 
     public static class Commands
@@ -318,6 +319,11 @@ public class AccordTestUtils
         return createTxn(key, key);
     }
 
+    public static Txn createTxn(Txn.Kind kind, Seekables<?, ?> seekables)
+    {
+        return AGENT.emptyTxn(kind, seekables);
+    }
+
     public static Ranges fullRange(Txn txn)
     {
         return fullRange(txn.keys());

test/unit/org/apache/cassandra/service/accord/CommandsForRangesTest.java

@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import accord.local.SaveStatus;
-import accord.primitives.Ranges;
-import accord.primitives.RoutableKey;
-import accord.primitives.TxnId;
-import accord.utils.AccordGens;
-import accord.utils.Gen;
-import accord.utils.Gens;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.Murmur3Partitioner;
-import org.apache.cassandra.dht.RandomPartitioner;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.utils.AccordGenerators;
-import org.apache.cassandra.utils.Interval;
-import org.apache.cassandra.utils.IntervalTree;
-
-import static accord.utils.Property.qt;
-import static org.apache.cassandra.simulator.RandomSource.Choices.choose;
-import static org.apache.cassandra.service.accord.AccordTestUtils.TABLE_ID1;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class CommandsForRangesTest
-{
-    private static Ranges FULL_RANGE = Ranges.of(new TokenRange(AccordRoutingKey.SentinelKey.min(TABLE_ID1), AccordRoutingKey.SentinelKey.max(TABLE_ID1)));
-
-    @BeforeClass
-    public static void setup() throws NoSuchFieldException, IllegalAccessException
-    {
-        DatabaseDescriptor.clientInitialization();
-        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
-    }
-
-    @Test
-    public void prune()
-    {
-        qt().forAll(cfr()).check(cfr -> {
-            // public void prune(TxnId pruneBefore, Ranges pruneRanges)
-            // private Timestamp maxRedundant;
-            List<TxnId> knownIds = new ArrayList<>(cfr.knownIds());
-            knownIds.sort(Comparator.naturalOrder());
-
-            assertThat(cfr.maxRedundant()).isNull();
-
-            TxnId min = knownIds.get(0);
-            TxnId max = knownIds.get(knownIds.size() - 1);
-
-            // should do nothing
-            IntervalTree<RoutableKey, CommandsForRanges.RangeCommandSummary, Interval<RoutableKey, CommandsForRanges.RangeCommandSummary>> tree = cfr.tree();
-            cfr.prune(min, FULL_RANGE);
-            assertThat(cfr.maxRedundant()).isNull();
-            assertThat(cfr.tree()).isEqualTo(tree);
-
-            cfr.prune(max, FULL_RANGE);
-            assertThat(cfr.knownIds()).containsExactly(max);
-            assertThat(cfr.maxRedundant()).isEqualTo(knownIds.size() == 1 ? null : knownIds.get(knownIds.size() - 2));
-
-            cfr.prune(new TxnId(max.logicalNext(max.node), max.kind(), max.domain()), FULL_RANGE);
-            assertThat(cfr.knownIds()).isEmpty();
-            assertThat(cfr.maxRedundant()).isEqualTo(max);
-        });
-    }
-
-    private static Gen<CommandsForRanges> cfr()
-    {
-        // TODO (coverage): once all partitioners work with regard to splitting, then should test all
-        Gen<IPartitioner> partitionerGen = rs -> choose(rs, Murmur3Partitioner.instance, RandomPartitioner.instance);
-        Gen<SaveStatus> statusGen = Gens.enums().all(SaveStatus.class);
-        return rs -> {
-            IPartitioner partitioner = partitionerGen.next(rs);
-            // some code reaches to the DD for partitioner...
-            DatabaseDescriptor.setPartitionerUnsafe(partitioner);
-            Gen<Ranges> rangesGen = AccordGenerators.ranges(ignore -> Collections.singleton(TABLE_ID1), ignore -> partitioner);
-            CommandsForRanges.Builder builder = new CommandsForRanges.Builder();
-            int numTxn = rs.nextInt(1, 10);
-            Set<TxnId> uniq = new HashSet<>();
-            for (int i = 0; i < numTxn; i++)
-            {
-                TxnId id;
-                while (!uniq.add(id = AccordGens.txnIds().next(rs))) {}
-                builder.put(id, rangesGen.next(rs), statusGen.next(rs), id, Collections.emptyList());
-            }
-            return builder.build();
-        };
-    }
-}

test/unit/org/apache/cassandra/service/accord/MockJournal.java

@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+import accord.local.SerializerSupport;
+import accord.messages.Accept;
+import accord.messages.Apply;
+import accord.messages.BeginRecovery;
+import accord.messages.Commit;
+import accord.messages.Message;
+import accord.messages.MessageType;
+import accord.messages.PreAccept;
+import accord.messages.Propagate;
+import accord.primitives.Ballot;
+import accord.primitives.TxnId;
+import org.agrona.collections.ObjectHashSet;
+
+import static accord.messages.MessageType.ACCEPT_REQ;
+import static accord.messages.MessageType.APPLY_MAXIMAL_REQ;
+import static accord.messages.MessageType.APPLY_MINIMAL_REQ;
+import static accord.messages.MessageType.BEGIN_RECOVER_REQ;
+import static accord.messages.MessageType.COMMIT_MAXIMAL_REQ;
+import static accord.messages.MessageType.COMMIT_SLOW_PATH_REQ;
+import static accord.messages.MessageType.PRE_ACCEPT_REQ;
+import static accord.messages.MessageType.PROPAGATE_APPLY_MSG;
+import static accord.messages.MessageType.PROPAGATE_PRE_ACCEPT_MSG;
+import static accord.messages.MessageType.PROPAGATE_STABLE_MSG;
+import static accord.messages.MessageType.STABLE_FAST_PATH_REQ;
+import static accord.messages.MessageType.STABLE_MAXIMAL_REQ;
+
+public class MockJournal implements IJournal
+{
+    private final Map<AccordJournal.Key, Message> writes = new HashMap<>();
+    @Override
+    public SerializerSupport.MessageProvider makeMessageProvider(TxnId txnId)
+    {
+        return new SerializerSupport.MessageProvider()
+        {
+            @Override
+            public Set<MessageType> test(Set<MessageType> messages)
+            {
+                Set<AccordJournal.Key> keys = new ObjectHashSet<>(messages.size() + 1, 0.9f);
+                for (MessageType message : messages)
+                    for (AccordJournal.Type synonymousType : AccordJournal.Type.synonymousTypesFromMessageType(message))
+                        keys.add(new AccordJournal.Key(txnId, synonymousType));
+                Set<AccordJournal.Key> presentKeys = Sets.intersection(writes.keySet(), keys);
+                Set<MessageType> presentMessages = new ObjectHashSet<>(presentKeys.size() + 1, 0.9f);
+                for (AccordJournal.Key key : presentKeys)
+                    presentMessages.add(key.type.outgoingType);
+                return presentMessages;
+            }
+
+            private <T extends Message> T get(AccordJournal.Key key)
+            {
+                return (T) writes.get(key);
+            }
+
+            private <T extends Message> T get(MessageType messageType)
+            {
+                for (AccordJournal.Type type : AccordJournal.Type.synonymousTypesFromMessageType(messageType))
+                {
+                    T value = get(new AccordJournal.Key(txnId, type));
+                    if (value != null) return value;
+                }
+                return null;
+            }
+
+            @Override
+            public PreAccept preAccept()
+            {
+                return get(PRE_ACCEPT_REQ);
+            }
+
+            @Override
+            public BeginRecovery beginRecover()
+            {
+                return get(BEGIN_RECOVER_REQ);
+            }
+
+            @Override
+            public Propagate propagatePreAccept()
+            {
+                return get(PROPAGATE_PRE_ACCEPT_MSG);
+            }
+
+            @Override
+            public Accept accept(Ballot ballot)
+            {
+                return get(ACCEPT_REQ);
+            }
+
+            @Override
+            public Commit commitSlowPath()
+            {
+                return get(COMMIT_SLOW_PATH_REQ);
+            }
+
+            @Override
+            public Commit commitMaximal()
+            {
+                return get(COMMIT_MAXIMAL_REQ);
+            }
+
+            @Override
+            public Commit stableFastPath()
+            {
+                return get(STABLE_FAST_PATH_REQ);
+            }
+
+            @Override
+            public Commit stableMaximal()
+            {
+                return get(STABLE_MAXIMAL_REQ);
+            }
+
+            @Override
+            public Propagate propagateStable()
+            {
+                return get(PROPAGATE_STABLE_MSG);
+            }
+
+            @Override
+            public Apply applyMinimal()
+            {
+                return get(APPLY_MINIMAL_REQ);
+            }
+
+            @Override
+            public Apply applyMaximal()
+            {
+                return get(APPLY_MAXIMAL_REQ);
+            }
+
+            @Override
+            public Propagate propagateApply()
+            {
+                return get(PROPAGATE_APPLY_MSG);
+            }
+        };
+    }
+
+    @Override
+    public void appendMessageBlocking(Message message)
+    {
+        AccordJournal.Type type = AccordJournal.Type.fromMessageType(message.type());
+        AccordJournal.Key key = new AccordJournal.Key(type.txnId(message), type);
+        writes.put(key, message);
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import org.junit.Test;
+
+import accord.local.PreLoadContext;
+import accord.local.SaveStatus;
+import accord.primitives.Ranges;
+import accord.primitives.TxnId;
+import accord.utils.AccordGens;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+
+public class SimpleSimulatedAccordCommandStoreTest extends SimulatedAccordCommandStoreTestBase
+{
+    @Test
+    public void emptyTxns()
+    {
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                for (int i = 0, examples = 100; i < examples; i++)
+                {
+                    TxnId id = AccordGens.txnIds().next(rs);
+                    instance.process(PreLoadContext.contextFor(id), (safe) -> {
+                        var safeCommand = safe.get(id, id, Ranges.EMPTY);
+                        var command = safeCommand.current();
+                        Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Uninitialised);
+                        return null;
+                    });
+                }
+            }
+
+        });
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java

@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import java.util.function.Function;
+import java.util.function.ToLongFunction;
+
+import accord.impl.SizeOfIntersectionSorter;
+import accord.impl.TestAgent;
+import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.CommandStores;
+import accord.local.Node;
+import accord.local.NodeTimeService;
+import accord.local.PreLoadContext;
+import accord.local.SafeCommandStore;
+import accord.messages.BeginRecovery;
+import accord.messages.Message;
+import accord.messages.PreAccept;
+import accord.messages.TxnRequest;
+import accord.primitives.Ballot;
+import accord.primitives.FullRoute;
+import accord.primitives.Keys;
+import accord.primitives.Ranges;
+import accord.primitives.Routable;
+import accord.primitives.RoutableKey;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.topology.Topologies;
+import accord.topology.Topology;
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.concurrent.ExecutorFactory;
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.SimulatedExecutorFactory;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.filter.ColumnFilter;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.metrics.AccordStateCacheMetrics;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Generators;
+import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
+import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
+import static org.apache.cassandra.utils.AccordGenerators.fromQT;
+
+class SimulatedAccordCommandStore implements AutoCloseable
+{
+    private final List<Throwable> failures = new ArrayList<>();
+    private final SimulatedExecutorFactory globalExecutor;
+    private final CommandStore.EpochUpdateHolder updateHolder;
+    private final BooleanSupplier shouldEvict, shouldFlush, shouldCompact;
+
+    public final NodeTimeService timeService;
+    public final AccordCommandStore store;
+    public final Node.Id nodeId;
+    public final Topology topology;
+    public final MockJournal journal;
+    public final ScheduledExecutorPlus unorderedScheduled;
+    public final List<String> evictions = new ArrayList<>();
+
+    SimulatedAccordCommandStore(RandomSource rs)
+    {
+        globalExecutor = new SimulatedExecutorFactory(accord.utilsfork.RandomSource.wrap(rs).fork(), fromQT(Generators.TIMESTAMP_GEN.map(java.sql.Timestamp::getTime)).mapToLong(TimeUnit.MILLISECONDS::toNanos).next(rs), failures::add);
+        this.unorderedScheduled = globalExecutor.scheduled("ignored");
+        ExecutorFactory.Global.unsafeSet(globalExecutor);
+        Stage.READ.unsafeSetExecutor(unorderedScheduled);
+        Stage.MUTATION.unsafeSetExecutor(unorderedScheduled);
+        for (Stage stage : Arrays.asList(Stage.MISC, Stage.ACCORD_MIGRATION))
+            stage.unsafeSetExecutor(globalExecutor.configureSequential("ignore").build());
+
+        this.updateHolder = new CommandStore.EpochUpdateHolder();
+        this.nodeId = AccordTopology.tcmIdToAccord(ClusterMetadata.currentNullable().myNodeId());
+        this.timeService = new NodeTimeService()
+        {
+            private final ToLongFunction<TimeUnit> unixWrapper = NodeTimeService.unixWrapper(TimeUnit.NANOSECONDS, this::now);
+
+            @Override
+            public Node.Id id()
+            {
+                return nodeId;
+            }
+
+            @Override
+            public long epoch()
+            {
+                return ClusterMetadata.current().epoch.getEpoch();
+            }
+
+            @Override
+            public long now()
+            {
+                return globalExecutor.nanoTime();
+            }
+
+            @Override
+            public long unix(TimeUnit unit)
+            {
+                return unixWrapper.applyAsLong(unit);
+            }
+
+            @Override
+            public Timestamp uniqueNow(Timestamp atLeast)
+            {
+                var now = Timestamp.fromValues(epoch(), now(), nodeId);
+                if (now.compareTo(atLeast) < 0)
+                    throw new UnsupportedOperationException();
+                return now;
+            }
+        };
+
+        this.journal = new MockJournal();
+        this.store = new AccordCommandStore(0,
+                                            timeService,
+                                            new TestAgent.RethrowAgent()
+                                            {
+                                                @Override
+                                                public boolean isExpired(TxnId initiated, long now)
+                                                {
+                                                    return false;
+                                                }
+                                            },
+                                            null,
+                                            ignore -> AccordTestUtils.NOOP_PROGRESS_LOG,
+                                            updateHolder,
+                                            journal,
+                                            new AccordStateCacheMetrics("test"));
+
+        store.cache().instances().forEach(i -> {
+            i.register(new AccordStateCache.Listener()
+            {
+                @Override
+                public void onAdd(AccordCachingState state)
+                {
+                }
+
+                @Override
+                public void onRelease(AccordCachingState state)
+                {
+                }
+
+                @Override
+                public void onEvict(AccordCachingState state)
+                {
+                    evictions.add(i + " evicted " + state);
+                }
+            });
+        });
+
+        this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
+        var rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges(), store);
+        updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
+        updateHolder.updateGlobal(topology.ranges());
+
+        shouldEvict = boolSource(rs.fork());
+        shouldFlush = boolSource(rs.fork());
+        shouldCompact = boolSource(rs.fork());
+    }
+
+    private static BooleanSupplier boolSource(RandomSource rs)
+    {
+        var gen = Gens.bools().mixedDistribution().next(rs);
+        return () -> gen.next(rs);
+    }
+
+    public TxnId nextTxnId(Txn.Kind kind, Routable.Domain domain)
+    {
+        return new TxnId(timeService.epoch(), timeService.now(), kind, domain, nodeId);
+    }
+
+    public void maybeCacheEvict(Keys keys, Ranges ranges)
+    {
+        AccordStateCache cache = store.cache();
+        cache.forEach(state -> {
+            Class<?> keyType = state.key().getClass();
+            if (TxnId.class.equals(keyType))
+            {
+                Command command = (Command) state.state().get();
+                if (command.known().definition.isKnown()
+                    && (command.partialTxn().keys().intersects(keys) || ranges.intersects(command.partialTxn().keys()))
+                    && shouldEvict.getAsBoolean())
+                    cache.maybeEvict(state);
+            }
+            else if (RoutableKey.class.isAssignableFrom(keyType))
+            {
+                RoutableKey key = (RoutableKey) state.key();
+                if ((keys.contains(key) || ranges.intersects(key))
+                    && shouldEvict.getAsBoolean())
+                    cache.maybeEvict(state);
+            }
+            else
+            {
+                throw new AssertionError("Unexpected key type: " + state.key().getClass());
+            }
+        });
+
+        for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+        {
+            Memtable memtable = store.getCurrentMemtable();
+            if (memtable.partitionCount() == 0 || !intersects(store, memtable, keys, ranges))
+                continue;
+            if (shouldFlush.getAsBoolean())
+                store.forceBlockingFlush(UNIT_TESTS);
+        }
+        for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+        {
+            if (store.getLiveSSTables().size() > 5 && shouldCompact.getAsBoolean())
+            {
+                // compaction no-op since auto-compaction is disabled... so need to enable quickly
+                store.enableAutoCompaction();
+                try
+                {
+                    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
+                }
+                finally
+                {
+                    store.disableAutoCompaction();
+                }
+            }
+        }
+    }
+
+    private static boolean intersects(ColumnFamilyStore store, Memtable memtable, Keys keys, Ranges ranges)
+    {
+        if (keys.isEmpty() && ranges.isEmpty()) // shouldn't happen, but just in case...
+            return false;
+        switch (store.name)
+        {
+            case "commands_for_key":
+                // pk = (store_id, key_token, key)
+                // since this is simulating a single store, store_id is a constant, so check key
+                try (var it = memtable.partitionIterator(ColumnFilter.NONE, DataRange.allData(store.getPartitioner()), null))
+                {
+                    while (it.hasNext())
+                    {
+                        var key = AccordKeyspace.CommandsForKeysAccessor.getKey(it.next().partitionKey());
+                        if (keys.contains(key) || ranges.intersects(key))
+                            return true;
+                    }
+                }
+                break;
+        }
+        return false;
+    }
+
+    public void checkFailures()
+    {
+        if (Thread.interrupted())
+            failures.add(new InterruptedException());
+        if (failures.isEmpty()) return;
+        AssertionError error = new AssertionError("Unexpected exceptions found");
+        failures.forEach(error::addSuppressed);
+        failures.clear();
+        throw error;
+    }
+
+    public <T> T process(TxnRequest<T> request) throws ExecutionException, InterruptedException
+    {
+        return process(request, request::apply);
+    }
+
+    public <T> T process(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function) throws ExecutionException, InterruptedException
+    {
+        var result = processAsync(loadCtx, function);
+        processAll();
+        return AsyncChains.getBlocking(result);
+    }
+
+    public <T> AsyncResult<T> processAsync(TxnRequest<T> request)
+    {
+        return processAsync(request, request::apply);
+    }
+
+    public <T> AsyncResult<T> processAsync(PreLoadContext loadCtx, Function<? super SafeCommandStore, T> function)
+    {
+        if (loadCtx instanceof Message)
+            journal.appendMessageBlocking((Message) loadCtx);
+        return store.submit(loadCtx, function).beginAsResult();
+    }
+
+    public Pair<TxnId, AsyncResult<PreAccept.PreAcceptOk>> enqueuePreAccept(Txn txn, FullRoute<?> route)
+    {
+        TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain());
+        PreAccept preAccept = new PreAccept(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology), txnId, txn, route);
+        return Pair.create(txnId, processAsync(preAccept, safe -> {
+            var reply = preAccept.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            return (PreAccept.PreAcceptOk) reply;
+        }));
+    }
+
+    public Pair<TxnId, AsyncResult<BeginRecovery.RecoverOk>> enqueueBeginRecovery(Txn txn, FullRoute<?> route)
+    {
+        TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain());
+        Ballot ballot = Ballot.fromValues(timeService.epoch(), timeService.now(), nodeId);
+        BeginRecovery br = new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology), txnId, txn, route, ballot);
+
+        return Pair.create(txnId, processAsync(br, safe -> {
+            var reply = br.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            return (BeginRecovery.RecoverOk) reply;
+        }).beginAsResult());
+    }
+
+    public void processAll()
+    {
+        while (processOne())
+        {
+        }
+    }
+
+    private boolean processOne()
+    {
+        boolean result = globalExecutor.processOne();
+        checkFailures();
+        return result;
+    }
+
+    @Override
+    public void close() throws Exception
+    {
+        store.shutdown();
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java

@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.collect.Maps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import accord.api.Key;
+import accord.impl.SizeOfIntersectionSorter;
+import accord.local.Node;
+import accord.messages.BeginRecovery;
+import accord.messages.PreAccept;
+import accord.primitives.Ballot;
+import accord.primitives.Deps;
+import accord.primitives.FullRoute;
+import accord.primitives.Keys;
+import accord.primitives.LatestDeps;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.topology.Topologies;
+import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
+
+public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester
+{
+    static
+    {
+        CassandraRelevantProperties.TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED.setBoolean(false);
+        // since this test does frequent truncates, the info table gets updated and forced flushed... which is 90% of the cost of this test...
+        // this flag disables that flush
+        CassandraRelevantProperties.UNSAFE_SYSTEM.setBoolean(true);
+        // The plan is to migrate away from SAI, so rather than hacking around timeout issues; just disable for now
+        CassandraRelevantProperties.SAI_TEST_DISABLE_TIMEOUT.setBoolean(true);
+    }
+
+    protected enum DepsMessage
+    {PreAccept, BeginRecovery, PreAcceptThenBeginRecovery}
+
+    protected static TableMetadata intTbl, reverseTokenTbl;
+    protected static Node.Id nodeId;
+
+    @BeforeClass
+    public static void setUpClass()
+    {
+        CQLTester.setUpClass();
+        DatabaseDescriptor.setIncrementalBackupsEnabled(false);
+    }
+
+    @Before
+    public void init()
+    {
+        if (intTbl != null)
+            return;
+        createKeyspace("CREATE KEYSPACE test WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }");
+        createTable("test", "CREATE TABLE test.tbl1 (pk int PRIMARY KEY, value int) WITH transactional_mode='full'");
+        intTbl = Schema.instance.getTableMetadata("test", "tbl1");
+
+        createTable("test", "CREATE TABLE test.tbl2 (pk vector<bigint, 2> PRIMARY KEY, value int) WITH transactional_mode='full'");
+        reverseTokenTbl = Schema.instance.getTableMetadata("test", "tbl2");
+
+        nodeId = AccordTopology.tcmIdToAccord(ClusterMetadata.current().myNodeId());
+
+        // tests may flush, which triggers compaction... since compaction is not simulated this adds a form of non-deterministic behavior
+        for (var store : Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
+            store.disableAutoCompaction();
+
+        AccordService.unsafeSetNoop();
+
+        ServerTestUtils.markCMS();
+    }
+
+    protected static void safeBlock(List<AsyncResult<?>> asyncs) throws InterruptedException, ExecutionException
+    {
+        int counter = 0;
+        for (var chain : asyncs)
+        {
+            Assertions.assertThat(chain.isDone())
+                      .describedAs("The %dth async task is blocked!", counter++)
+                      .isTrue();
+            AsyncChains.getBlocking(chain);
+        }
+    }
+
+    protected static void safeBlock(List<AsyncResult<?>> asyncs, List<?> details) throws InterruptedException, ExecutionException
+    {
+        int counter = 0;
+        for (var chain : asyncs)
+        {
+            Assertions.assertThat(chain.isDone())
+                      .describedAs("The %dth async task %s is blocked!", counter, details.get(counter++))
+                      .isTrue();
+            AsyncChains.getBlocking(chain);
+        }
+    }
+
+    protected static TokenRange fullRange(TableId id)
+    {
+        return new TokenRange(AccordRoutingKey.SentinelKey.min(id), AccordRoutingKey.SentinelKey.max(id));
+    }
+
+    protected static TokenRange tokenRange(TableId id, long start, long end)
+    {
+        return new TokenRange(start == Long.MIN_VALUE ? AccordRoutingKey.SentinelKey.min(id) : tokenKey(id, start), tokenKey(id, end));
+    }
+
+    protected static AccordRoutingKey.TokenKey tokenKey(TableId id, long token)
+    {
+        return new AccordRoutingKey.TokenKey(id, new Murmur3Partitioner.LongToken(token));
+    }
+
+    protected static Map<Key, List<TxnId>> keyConflicts(List<TxnId> list, Keys keys)
+    {
+        Map<Key, List<TxnId>> kc = Maps.newHashMapWithExpectedSize(keys.size());
+        for (Key key : keys)
+        {
+            if (list.isEmpty())
+                continue;
+            kc.put(key, list);
+        }
+        return kc;
+    }
+
+    protected static Map<Range, List<TxnId>> rangeConflicts(List<TxnId> list, Ranges ranges)
+    {
+        Map<Range, List<TxnId>> kc = Maps.newHashMapWithExpectedSize(ranges.size());
+        for (Range range : ranges)
+        {
+            if (list.isEmpty())
+                continue;
+            kc.put(range, list);
+        }
+        return kc;
+    }
+
+    protected static TxnId assertDepsMessage(SimulatedAccordCommandStore instance,
+                                             DepsMessage messageType,
+                                             Txn txn, FullRoute<?> route,
+                                             Map<Key, List<TxnId>> keyConflicts) throws ExecutionException, InterruptedException
+    {
+        return assertDepsMessage(instance, messageType, txn, route, keyConflicts, Collections.emptyMap());
+    }
+
+    protected static TxnId assertDepsMessage(SimulatedAccordCommandStore instance,
+                                             DepsMessage messageType,
+                                             Txn txn, FullRoute<?> route,
+                                             Map<Key, List<TxnId>> keyConflicts,
+                                             Map<Range, List<TxnId>> rangeConflicts) throws ExecutionException, InterruptedException
+    {
+        var pair = assertDepsMessageAsync(instance, messageType, txn, route, keyConflicts, rangeConflicts);
+        instance.processAll();
+        AsyncChains.getBlocking(pair.right);
+
+        return pair.left;
+    }
+
+    protected static Pair<TxnId, AsyncResult<?>> assertDepsMessageAsync(SimulatedAccordCommandStore instance,
+                                                                        DepsMessage messageType,
+                                                                        Txn txn, FullRoute<?> route,
+                                                                        Map<Key, List<TxnId>> keyConflicts)
+    {
+        return assertDepsMessageAsync(instance, messageType, txn, route, keyConflicts, Collections.emptyMap());
+    }
+
+    protected static Pair<TxnId, AsyncResult<?>> assertDepsMessageAsync(SimulatedAccordCommandStore instance,
+                                                                        DepsMessage messageType,
+                                                                        Txn txn, FullRoute<?> route,
+                                                                        Map<Key, List<TxnId>> keyConflicts,
+                                                                        Map<Range, List<TxnId>> rangeConflicts)
+    {
+        switch (messageType)
+        {
+            case PreAccept:
+                return assertPreAcceptAsync(instance, txn, route, keyConflicts, rangeConflicts);
+            case BeginRecovery:
+                return assertBeginRecoveryAsync(instance, txn, route, keyConflicts, rangeConflicts);
+            case PreAcceptThenBeginRecovery:
+                return assertBeginRecoveryAfterPreAcceptAsync(instance, txn, route, keyConflicts, rangeConflicts);
+            default:
+                throw new IllegalArgumentException("Unknown message type: " + messageType);
+        }
+    }
+
+    protected static Pair<TxnId, AsyncResult<?>> assertPreAcceptAsync(SimulatedAccordCommandStore instance,
+                                                                      Txn txn, FullRoute<?> route,
+                                                                      Map<Key, List<TxnId>> keyConflicts,
+                                                                      Map<Range, List<TxnId>> rangeConflicts)
+    {
+        Map<Key, List<TxnId>> cloneKeyConflicts = keyConflicts.entrySet().stream()
+                                                              .filter(e -> !e.getValue().isEmpty())
+                                                              .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+        Map<Range, List<TxnId>> cloneRangeConflicts = rangeConflicts.entrySet().stream()
+                                                                    .filter(e -> !e.getValue().isEmpty())
+                                                                    .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+        var pair = instance.enqueuePreAccept(txn, route);
+        return Pair.create(pair.left, pair.right.map(success -> {
+            assertDeps(success.txnId, success.deps, cloneKeyConflicts, cloneRangeConflicts);
+            return null;
+        }).beginAsResult());
+    }
+
+    protected static Pair<TxnId, AsyncResult<?>> assertBeginRecoveryAsync(SimulatedAccordCommandStore instance,
+                                                                          Txn txn, FullRoute<?> route,
+                                                                          Map<Key, List<TxnId>> keyConflicts,
+                                                                          Map<Range, List<TxnId>> rangeConflicts)
+    {
+        Map<Key, List<TxnId>> cloneKeyConflicts = keyConflicts.entrySet().stream()
+                                                              .filter(e -> !e.getValue().isEmpty())
+                                                              .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+        Map<Range, List<TxnId>> cloneRangeConflicts = rangeConflicts.entrySet().stream()
+                                                                    .filter(e -> !e.getValue().isEmpty())
+                                                                    .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+        var pair = instance.enqueueBeginRecovery(txn, route);
+        return Pair.create(pair.left, pair.right.map(success -> {
+            Deps proposeDeps = LatestDeps.mergeProposal(Collections.singletonList(success), ok -> ok.deps);
+            assertDeps(success.txnId, proposeDeps, cloneKeyConflicts, cloneRangeConflicts);
+            return null;
+        }).beginAsResult());
+    }
+
+    protected static Pair<TxnId, AsyncResult<?>> assertBeginRecoveryAfterPreAcceptAsync(SimulatedAccordCommandStore instance,
+                                                                                        Txn txn, FullRoute<?> route,
+                                                                                        Map<Key, List<TxnId>> keyConflicts,
+                                                                                        Map<Range, List<TxnId>> rangeConflicts)
+    {
+        Map<Key, List<TxnId>> cloneKeyConflicts = keyConflicts.entrySet().stream()
+                                                              .filter(e -> !e.getValue().isEmpty())
+                                                              .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+        Map<Range, List<TxnId>> cloneRangeConflicts = rangeConflicts.entrySet().stream()
+                                                                    .filter(e -> !e.getValue().isEmpty())
+                                                                    .collect(Collectors.toMap(e -> e.getKey(), e -> new ArrayList(e.getValue())));
+
+        TxnId txnId = instance.nextTxnId(txn.kind(), txn.keys().domain());
+        PreAccept preAccept = new PreAccept(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, txn, route);
+
+        var preAcceptAsync = instance.processAsync(preAccept, safe -> {
+            var reply = preAccept.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            PreAccept.PreAcceptOk success = (PreAccept.PreAcceptOk) reply;
+            assertDeps(success.txnId, success.deps, cloneKeyConflicts, cloneRangeConflicts);
+            return success;
+        });
+        var delay = preAcceptAsync.flatMap(ignore -> AsyncChains.ofCallable(instance.unorderedScheduled, () -> {
+            Ballot ballot = Ballot.fromValues(instance.timeService.epoch(), instance.timeService.now(), nodeId);
+            return new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, txn, route, ballot);
+        }));
+        var recoverAsync = delay.flatMap(br -> instance.processAsync(br, safe -> {
+            var reply = br.apply(safe);
+            Assertions.assertThat(reply.isOk()).isTrue();
+            BeginRecovery.RecoverOk success = (BeginRecovery.RecoverOk) reply;
+            Deps proposeDeps = LatestDeps.mergeProposal(Collections.singletonList(success), ok -> ok.deps);
+            assertDeps(success.txnId, proposeDeps, cloneKeyConflicts, cloneRangeConflicts);
+            return success;
+        }));
+
+        return Pair.create(txnId, recoverAsync.beginAsResult());
+    }
+
+    protected static void assertDeps(TxnId txnId, Deps deps,
+                                     Map<Key, List<TxnId>> keyConflicts,
+                                     Map<Range, List<TxnId>> rangeConflicts)
+    {
+        if (rangeConflicts.isEmpty())
+        {
+            Assertions.assertThat(deps.rangeDeps.isEmpty()).describedAs("Txn %s rangeDeps was not empty; %s", txnId, deps.rangeDeps).isTrue();
+        }
+        else
+        {
+            List<Range> actualRanges = IntStream.range(0, deps.rangeDeps.rangeCount()).mapToObj(i -> deps.rangeDeps.range(i)).collect(Collectors.toList());
+//            Assertions.assertThat(deps.rangeDeps.rangeCount()).describedAs("Txn %s Expected ranges size; %s", txnId, deps.rangeDeps).isEqualTo(rangeConflicts.size());
+            Assertions.assertThat(Ranges.of(actualRanges.toArray(Range[]::new)))
+                      .describedAs("Txn %s had different ranges than expected", txnId)
+                      .isEqualTo(Ranges.of(rangeConflicts.keySet().toArray(Range[]::new)));
+            AssertionError errors = null;
+            for (int i = 0; i < rangeConflicts.size(); i++)
+            {
+                try
+                {
+                    var range = deps.rangeDeps.range(i);
+                    Assertions.assertThat(rangeConflicts).describedAs("Txn %s had an unexpected range", txnId).containsKey(range);
+                    var conflict = deps.rangeDeps.txnIdsForRangeIndex(i);
+                    List<TxnId> expectedConflict = rangeConflicts.get(range);
+                    Assertions.assertThat(conflict).describedAs("Txn %s Expected range %s to have different conflicting txns", txnId, range).isEqualTo(expectedConflict);
+                }
+                catch (AssertionError e)
+                {
+                    if (errors == null)
+                        errors = e;
+                    else
+                        errors.addSuppressed(e);
+                }
+            }
+            if (errors != null)
+                throw errors;
+        }
+        if (keyConflicts.isEmpty())
+        {
+            Assertions.assertThat(deps.keyDeps.isEmpty()).describedAs("Txn %s keyDeps was not empty", txnId).isTrue();
+        }
+        else
+        {
+            Assertions.assertThat(deps.keyDeps.keys()).describedAs("Txn %s Keys", txnId).isEqualTo(Keys.of(keyConflicts.keySet()));
+            for (var key : keyConflicts.keySet())
+                Assertions.assertThat(deps.keyDeps.txnIds(key)).describedAs("Txn %s for key %s", txnId, key).isEqualTo(keyConflicts.get(key));
+        }
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java

@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import accord.api.Key;
+import accord.primitives.FullKeyRoute;
+import accord.primitives.FullRangeRoute;
+import accord.primitives.FullRoute;
+import accord.primitives.Keys;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
+
+public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
+{
+    @Test
+    public void keyConflicts()
+    {
+        TableMetadata tbl = intTbl;
+        int numSamples = 100;
+
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            int key = rs.nextInt();
+            PartitionKey pk = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(Int32Type.instance.decompose(key)));
+            Keys keys = Keys.of(pk);
+            FullKeyRoute route = keys.toRoute(pk.toUnseekable());
+            Txn txn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                List<TxnId> conflicts = new ArrayList<>(numSamples);
+                boolean concurrent = rs.nextBoolean();
+                List<AsyncResult<?>> asyncs = !concurrent ? null : new ArrayList<>(numSamples);
+                for (int i = 0; i < numSamples; i++)
+                {
+                    instance.maybeCacheEvict(keys, Ranges.EMPTY);
+                    if (concurrent)
+                    {
+                        var pair = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), txn, route, keyConflicts(conflicts, keys));
+                        conflicts.add(pair.left);
+                        asyncs.add(pair.right);
+                    }
+                    else
+                    {
+                        conflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), txn, route, keyConflicts(conflicts, keys)));
+                    }
+                }
+                if (concurrent)
+                {
+                    instance.processAll();
+                    safeBlock(asyncs);
+                }
+            }
+        });
+    }
+
+    @Test
+    public void concurrentRangePartialKeyMatch()
+    {
+        var tbl = reverseTokenTbl;
+        int numSamples = 250;
+        int numConflictKeyTxns = 10;
+
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                long token = rs.nextLong(Long.MIN_VALUE  + 1, Long.MAX_VALUE);
+                Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - 1, token));
+                long outOfRangeToken = token - 10;
+                if (outOfRangeToken == Long.MIN_VALUE) // if this wraps around that is fine, just can't be min
+                    outOfRangeToken++;
+                Key key = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(LongToken.keyForToken(token)));
+                Key outOfRangeKey = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(LongToken.keyForToken(outOfRangeToken)));
+                Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)",
+                                                 "INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"),
+                                       Arrays.asList(LongToken.keyForToken(token), 42,
+                                                     LongToken.keyForToken(outOfRangeToken), 42));
+                Keys keys = (Keys) keyTxn.keys();
+                FullRoute<?> keyRoute = keys.toRoute(keys.get(0).toUnseekable());
+
+                Txn conflictingKeyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"),
+                                                  Arrays.asList(LongToken.keyForToken(outOfRangeToken), 42));
+                Keys conflictingKeys = (Keys) conflictingKeyTxn.keys();
+                FullRoute<?> conflictingRoute = conflictingKeys.toRoute(conflictingKeys.get(0).toUnseekable());
+
+                FullRangeRoute rangeRoute = partialRange.toRoute(keys.get(0).toUnseekable());
+                Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, partialRange);
+
+                List<TxnId> keyConflicts = new ArrayList<>(numSamples);
+                List<TxnId> outOfRangeKeyConflicts = new ArrayList<>(numSamples);
+                List<TxnId> rangeConflicts = new ArrayList<>(numSamples);
+                List<AsyncResult<?>> asyncs = new ArrayList<>(numSamples * 2 + numSamples * numConflictKeyTxns);
+                List<TxnId> asyncIds = new ArrayList<>(numSamples * 2 + numSamples * numConflictKeyTxns);
+                for (int i = 0; i < numSamples; i++)
+                {
+                    instance.maybeCacheEvict((Keys) keyTxn.keys(), partialRange);
+                    for (int j = 0; j < numConflictKeyTxns; j++)
+                    {
+                        var p = instance.enqueuePreAccept(conflictingKeyTxn, conflictingRoute);
+                        outOfRangeKeyConflicts.add(p.left);
+                        asyncs.add(p.right);
+                        asyncIds.add(p.left);
+                    }
+
+                    var k = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, Map.of(key, keyConflicts, outOfRangeKey, outOfRangeKeyConflicts), Collections.emptyMap());
+                    keyConflicts.add(k.left);
+                    outOfRangeKeyConflicts.add(k.left);
+                    asyncs.add(k.right);
+                    asyncIds.add(k.left);
+
+                    var r = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, Map.of(key, keyConflicts), rangeConflicts(rangeConflicts, partialRange));
+                    rangeConflicts.add(r.left);
+                    asyncs.add(r.right);
+                    asyncIds.add(r.left);
+                }
+                instance.processAll();
+                safeBlock(asyncs, asyncIds);
+            }
+        });
+    }
+
+    @Test
+    public void simpleRangeConflicts()
+    {
+        var tbl = reverseTokenTbl;
+        Ranges wholeRange = Ranges.of(fullRange(tbl.id));
+        int numSamples = 100;
+
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                long token = rs.nextLong(Long.MIN_VALUE  + 1, Long.MAX_VALUE);
+                ByteBuffer key = LongToken.keyForToken(token);
+                PartitionKey pk = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(key));
+                Keys keys = Keys.of(pk);
+                FullKeyRoute keyRoute = keys.toRoute(pk.toUnseekable());
+                Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
+
+                Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - 1, token));
+                boolean useWholeRange = rs.nextBoolean();
+                Ranges ranges = useWholeRange ? wholeRange : partialRange;
+                FullRangeRoute rangeRoute = ranges.toRoute(pk.toUnseekable());
+                Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, ranges);
+
+                List<TxnId> keyConflicts = new ArrayList<>(numSamples);
+                List<TxnId> rangeConflicts = new ArrayList<>(numSamples);
+                boolean concurrent = rs.nextBoolean();
+                List<AsyncResult<?>> asyncs = !concurrent ? null : new ArrayList<>(numSamples * 2);
+                for (int i = 0; i < numSamples; i++)
+                {
+                    instance.maybeCacheEvict(keys, ranges);
+                    if (concurrent)
+                    {
+                        var k = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, keyConflicts(keyConflicts, keys));
+                        keyConflicts.add(k.left);
+                        asyncs.add(k.right);
+                        var r = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts(keyConflicts, keys), rangeConflicts(rangeConflicts, ranges));
+                        rangeConflicts.add(r.left);
+                        asyncs.add(r.right);
+                    }
+                    else
+                    {
+                        keyConflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, keyConflicts(keyConflicts, keys)));
+                        rangeConflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts(keyConflicts, keys), rangeConflicts(rangeConflicts, ranges)));
+                    }
+                }
+                if (concurrent)
+                {
+                    instance.processAll();
+                    safeBlock(asyncs);
+                }
+            }
+        });
+    }
+
+    @Test
+    public void expandingRangeConflicts()
+    {
+        var tbl = reverseTokenTbl;
+        int numSamples = 100;
+
+        qt().withSeed(6484101342775432632L).withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                long token = rs.nextLong(Long.MIN_VALUE + numSamples + 1, Long.MAX_VALUE - numSamples);
+                ByteBuffer key = LongToken.keyForToken(token);
+                PartitionKey pk = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(key));
+                Keys keys = Keys.of(pk);
+                FullKeyRoute keyRoute = keys.toRoute(pk.toUnseekable());
+                Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
+
+                List<TxnId> keyConflicts = new ArrayList<>(numSamples);
+                Map<Range, List<TxnId>> rangeConflicts = new HashMap<>();
+                boolean concurrent = rs.nextBoolean();
+                List<AsyncResult<?>> asyncs = !concurrent ? null : new ArrayList<>(numSamples);
+                List<TxnId> info = !concurrent ? null : new ArrayList<>(numSamples);
+                for (int i = 0; i < numSamples; i++)
+                {
+                    Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - i - 1, token + i));
+                    FullRangeRoute rangeRoute = partialRange.toRoute(pk.toUnseekable());
+                    Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, partialRange);
+                    try
+                    {
+                        instance.maybeCacheEvict(keys, partialRange);
+                        if (concurrent)
+                        {
+                            var pair = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, keyConflicts(keyConflicts, keys));
+                            info.add(pair.left);
+                            keyConflicts.add(pair.left);
+                            asyncs.add(pair.right);
+
+                            pair = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts(keyConflicts, keys), rangeConflicts);
+                            info.add(pair.left);
+                            rangeConflicts.put(partialRange.get(0), Collections.singletonList(pair.left));
+                            asyncs.add(pair.right);
+                        }
+                        else
+                        {
+                            keyConflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, keyConflicts(keyConflicts, keys)));
+                            rangeConflicts.put(partialRange.get(0), Collections.singletonList(assertDepsMessage(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts(keyConflicts, keys), rangeConflicts)));
+                        }
+                    }
+                    catch (Throwable t)
+                    {
+                        AssertionError error = new AssertionError("Unexpected error: i=" + i + ", token=" + token + ", range=" + partialRange.get(0));
+                        t.addSuppressed(error);
+                        throw t;
+                    }
+                }
+                if (concurrent)
+                {
+                    instance.processAll();
+                    safeBlock(asyncs, info);
+                }
+            }
+        });
+    }
+
+    @Test
+    public void overlappingRangeConflicts()
+    {
+        var tbl = reverseTokenTbl;
+        int numSamples = 100;
+
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                long token = rs.nextLong(Long.MIN_VALUE + numSamples + 1, Long.MAX_VALUE - numSamples);
+                ByteBuffer key = LongToken.keyForToken(token);
+                PartitionKey pk = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(key));
+                Keys keys = Keys.of(pk);
+                FullKeyRoute keyRoute = keys.toRoute(pk.toUnseekable());
+                Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
+
+                Range left = tokenRange(tbl.id, token - 10, token + 5);
+                Range right = tokenRange(tbl.id, token - 5, token + 10);
+
+                List<TxnId> keyConflicts = new ArrayList<>(numSamples);
+                Map<Range, List<TxnId>> rangeConflicts = new HashMap<>();
+                rangeConflicts.put(left, new ArrayList<>());
+                rangeConflicts.put(right, new ArrayList<>());
+                for (int i = 0; i < numSamples; i++)
+                {
+                    Ranges partialRange = Ranges.of(rs.nextBoolean() ? left : right);
+                    try
+                    {
+                        instance.maybeCacheEvict(keys, partialRange);
+                        keyConflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, keyConflicts(keyConflicts, keys)));
+
+                        FullRangeRoute rangeRoute = partialRange.toRoute(pk.toUnseekable());
+                        Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, partialRange);
+                        rangeConflicts.get(partialRange.get(0)).add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts(keyConflicts, keys), rangeConflicts));
+                    }
+                    catch (Throwable t)
+                    {
+                        AssertionError error = new AssertionError("Unexpected error: i=" + i + ", token=" + token + ", range=" + partialRange.get(0));
+                        t.addSuppressed(error);
+                        throw t;
+                    }
+                }
+            }
+        });
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java

@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+
+import accord.api.Key;
+import accord.api.RoutingKey;
+import accord.primitives.FullRangeRoute;
+import accord.primitives.FullRoute;
+import accord.primitives.Keys;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.utils.RTree;
+import org.apache.cassandra.utils.RangeTree;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.dht.Murmur3Partitioner.LongToken.keyForToken;
+import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
+
+public class SimulatedMultiKeyAndRangeTest extends SimulatedAccordCommandStoreTestBase
+{
+    @Test
+    public void test()
+    {
+        var tbl = reverseTokenTbl;
+        int numSamples = 300;
+        long minToken = 0;
+        long maxToken = 100;
+        Gen<Gen.LongGen> tokenDistribution = Gens.mixedDistribution(minToken, maxToken + 1);
+        Gen<Gen.IntGen> keyDistribution = Gens.mixedDistribution(1, 5);
+        Gen<Gen.IntGen> rangeDistribution = Gens.mixedDistribution(1, 5);
+        Gen<Gen<Domain>> domainDistribution = Gens.mixedDistribution(Domain.values());
+        Gen<Gen<DepsMessage>> msgDistribution = Gens.mixedDistribution(DepsMessage.values());
+
+        qt().withExamples(100).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                Gen.LongGen tokenGen = tokenDistribution.next(rs);
+                Gen<Domain> domainGen = domainDistribution.next(rs);
+                Gen<DepsMessage> msgGen = msgDistribution.next(rs);
+                Map<Key, List<TxnId>> keyConflicts = new HashMap<>();
+                RangeTree<RoutingKey, Range, TxnId> rangeConflicts = RTree.create(RangeTreeRangeAccessor.instance);
+                List<AsyncResult<?>> asyncs = new ArrayList<>(numSamples);
+
+                Gen.IntGen keyCountGen = keyDistribution.next(rs);
+                Gen.IntGen rangeCountGen = rangeDistribution.next(rs);
+
+                for (int i = 0; i < numSamples; i++)
+                {
+                    switch (domainGen.next(rs))
+                    {
+                        case Key:
+                        {
+                            int numKeys = keyCountGen.nextInt(rs);
+                            TreeSet<Key> set = new TreeSet<>();
+                            while (set.size() != numKeys)
+                                set.add(new PartitionKey(tbl.id, tbl.partitioner.decorateKey(keyForToken(tokenGen.nextLong(rs)))));
+                            Keys keys = Keys.of(set);
+                            List<String> inserts = IntStream.range(0, numKeys).mapToObj(ignore -> "INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)").collect(Collectors.toList());
+                            List<Object> binds = new ArrayList<>(numKeys * 2);
+                            keys.forEach(k -> {
+                                binds.add(((PartitionKey) k.asKey()).partitionKey().getKey());
+                                binds.add(42);
+                            });
+                            Txn txn = createTxn(wrapInTxn(inserts), binds);
+                            FullRoute<?> route = keys.toRoute(keys.get(0).toUnseekable());
+
+                            Map<Key, List<TxnId>> expectedConflicts = new HashMap<>();
+                            keys.forEach(k -> expectedConflicts.put(k, keyConflicts.computeIfAbsent(k, ignore -> new ArrayList<>())));
+
+                            var p = assertDepsMessageAsync(instance, msgGen.next(rs), txn, route, expectedConflicts, Collections.emptyMap());
+                            keys.forEach(k -> keyConflicts.get(k).add(p.left));
+                            asyncs.add(p.right);
+                        }
+                        break;
+                        case Range:
+                        {
+                            int numRanges = rangeCountGen.nextInt(rs);
+                            Set<Range> set = new HashSet<>();
+                            while (set.size() != numRanges)
+                            {
+                                long token = tokenGen.nextLong(rs);
+                                int offset = rs.nextInt(1, 10);
+                                long start, end;
+                                if (token + offset > maxToken)
+                                {
+                                    end = token;
+                                    start = end - offset;
+                                }
+                                else
+                                {
+                                    start = token;
+                                    end = start + offset;
+                                }
+                                set.add(tokenRange(tbl.id, start, end));
+                            }
+                            // The property ranges.size() == numRanges is not true as this logic will sort + deoverlap
+                            // so if the ranges were overlapped, we could have more or less than numRanges
+                            Ranges ranges = Ranges.of(set.toArray(Range[]::new));
+                            FullRangeRoute route = ranges.toRoute(ranges.get(0).end());
+                            Txn txn = createTxn(Txn.Kind.ExclusiveSyncPoint, ranges);
+
+                            Map<Key, List<TxnId>> expectedKeyConflicts = keyConflicts.entrySet().stream()
+                                                                                     .filter(e -> ranges.contains(e.getKey()))
+                                                                                     .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                            Map<Range, List<TxnId>> expectedRangeConflicts = new HashMap<>();
+                            ranges.forEach(r ->
+                                           rangeConflicts.search(r, e ->
+                                                                    expectedRangeConflicts.computeIfAbsent(e.getKey(), ignore -> new ArrayList<>()).add(e.getValue())));
+                            // need to dedup/sort txns
+                            expectedRangeConflicts.values().forEach(l -> {
+                                var sortedDedup = new ArrayList<>(new TreeSet<>(l));
+                                l.clear();
+                                l.addAll(sortedDedup);
+                            });
+                            var p = assertDepsMessageAsync(instance, msgGen.next(rs), txn, route, expectedKeyConflicts, expectedRangeConflicts);
+                            asyncs.add(p.right);
+                            ranges.forEach(r -> rangeConflicts.add(r, p.left));
+                        }
+                        break;
+                        default:
+                            throw new AssertionError();
+                    }
+                }
+                instance.processAll();
+                safeBlock(asyncs);
+            }
+        });
+    }
+}

test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+import accord.api.Key;
+import accord.primitives.FullRangeRoute;
+import accord.primitives.FullRoute;
+import accord.primitives.Keys;
+import accord.primitives.Ranges;
+import accord.primitives.Txn;
+import accord.primitives.TxnId;
+import accord.utils.async.AsyncResult;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.dht.Murmur3Partitioner.LongToken.keyForToken;
+import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
+
+public class SimulatedRandomKeysWithRangeConflictTest extends SimulatedAccordCommandStoreTestBase
+{
+    @Test
+    public void keysAllOverConflictingWithRange()
+    {
+        var tbl = reverseTokenTbl;
+        Ranges wholeRange = Ranges.of(fullRange(tbl.id));
+        FullRangeRoute rangeRoute = wholeRange.toRoute(wholeRange.get(0).end());
+        Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, wholeRange);
+        int numSamples = 300;
+
+        qt().withExamples(10).check(rs -> {
+            AccordKeyspace.unsafeClear();
+            try (var instance = new SimulatedAccordCommandStore(rs))
+            {
+                Map<Key, List<TxnId>> keyConflicts = new HashMap<>();
+                List<TxnId> rangeConflicts = new ArrayList<>(numSamples);
+                boolean concurrent = rs.nextBoolean();
+                List<AsyncResult<?>> asyncs = !concurrent ? null : new ArrayList<>(numSamples * 2);
+                for (int i = 0; i < numSamples; i++)
+                {
+                    long token = rs.nextLong(Long.MIN_VALUE  + 1, Long.MAX_VALUE);
+                    Key key = new PartitionKey(tbl.id, tbl.partitioner.decorateKey(keyForToken(token)));
+                    Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"),
+                                           Arrays.asList(keyForToken(token), 42));
+                    Keys keys = (Keys) keyTxn.keys();
+                    FullRoute<?> keyRoute = keys.toRoute(keys.get(0).toUnseekable());
+
+                    instance.maybeCacheEvict((Keys) keyTxn.keys(), wholeRange);
+
+                    if (concurrent)
+                    {
+                        var k = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, Map.of(key, keyConflicts.computeIfAbsent(key, ignore -> new ArrayList<>())), Collections.emptyMap());
+                        keyConflicts.get(key).add(k.left);
+                        asyncs.add(k.right);
+
+                        var r = assertDepsMessageAsync(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts, rangeConflicts(rangeConflicts, wholeRange));
+                        rangeConflicts.add(r.left);
+                        asyncs.add(r.right);
+                    }
+                    else
+                    {
+                        var k = assertDepsMessage(instance, rs.pick(DepsMessage.values()), keyTxn, keyRoute, Map.of(key, keyConflicts.computeIfAbsent(key, ignore -> new ArrayList<>())), Collections.emptyMap());
+                        keyConflicts.get(key).add(k);
+                        rangeConflicts.add(assertDepsMessage(instance, rs.pick(DepsMessage.values()), rangeTxn, rangeRoute, keyConflicts, rangeConflicts(rangeConflicts, wholeRange)));
+                    }
+                }
+                if (concurrent)
+                {
+                    instance.processAll();
+                    safeBlock(asyncs);
+                }
+            }
+        });
+    }
+}

test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java

@@ -26,8 +26,12 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
 
+import accord.utils.DefaultRandom;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
+import org.apache.cassandra.concurrent.SimulatedExecutorFactory;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -101,6 +105,11 @@ public class AsyncOperationTest
     private static final Logger logger = LoggerFactory.getLogger(AsyncOperationTest.class);
     private static final AtomicLong clock = new AtomicLong(0);
 
+    static
+    {
+        CassandraRelevantProperties.TEST_ACCORD_STORE_THREAD_CHECKS_ENABLED.setBoolean(false);
+    }
+
     @BeforeClass
     public static void beforeClass() throws Throwable
     {
@@ -314,7 +323,8 @@ public class AsyncOperationTest
     @Test
     public void testFutureCleanup() throws Throwable
     {
-        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
+        SimulatedExecutorFactory factory = new SimulatedExecutorFactory(accord.utilsfork.RandomSource.wrap(new DefaultRandom(42)), 42);
+        AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", factory.scheduled("ignored"), Stage.MUTATION.executor());
 
         TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
 
@@ -342,12 +352,15 @@ public class AsyncOperationTest
                         {
                             case SETUP:
                                 assertFutureState(cache(), txnId, false, false, false);
+                                factory.processAll();
                                 break;
                             case FINISHED:
                                 assertFutureState(cache(), txnId, true, false, false);
+                                factory.processAll();
                                 break;
                             case LOADING:
                                 assertFutureState(cache(), txnId, true, true, false);
+                                factory.processAll();
                                 break;
                         }
                         super.state(state);

test/unit/org/apache/cassandra/service/accord/serializers/AccordRoutingKeyByteSourceTest.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource.FixedLength;
+import org.apache.cassandra.utils.AccordGenerators;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.CassandraGenerators;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+import static org.apache.cassandra.utils.AccordGenerators.fromQT;
+import static org.apache.cassandra.utils.CassandraGenerators.token;
+
+public class AccordRoutingKeyByteSourceTest
+{
+    static
+    {
+        DatabaseDescriptor.clientInitialization();
+        // AccordRoutingKey$TokenKey reaches into DD to get partitioner, so need to set that up...
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+    }
+
+    @Test
+    public void tokenSerde()
+    {
+        qt().forAll(fromQT(token())).check(token -> {
+            var serializer = AccordRoutingKeyByteSource.create(token.getPartitioner());
+            byte[] min = ByteSourceInverse.readBytes(serializer.minAsComparableBytes());
+            byte[] max = ByteSourceInverse.readBytes(serializer.maxAsComparableBytes());
+
+            var bytes = serializer.serialize(token);
+            if (serializer instanceof FixedLength)
+            {
+                FixedLength fl = (FixedLength) serializer;
+                Assertions.assertThat(bytes)
+                          .hasSize(fl.valueSize())
+                          .hasSize(min.length)
+                          .hasSize(max.length);
+            }
+
+            Assertions.assertThat(ByteArrayUtil.compareUnsigned(min, 0, bytes, 0, bytes.length)).isLessThan(0);
+            Assertions.assertThat(ByteArrayUtil.compareUnsigned(max, 0, bytes, 0, bytes.length)).isGreaterThan(0);
+
+            var read = serializer.tokenFromComparableBytes(ByteArrayAccessor.instance, bytes);
+            Assertions.assertThat(read).isEqualTo(token);
+        });
+    }
+
+    @Test
+    public void accordRoutingKeySerde()
+    {
+        qt().forAll(AccordGenerators.routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(token()))).check(key -> {
+            AccordRoutingKeyByteSource.Serializer serializer = key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL ?
+                                                               // doesn't really matter...
+                                                               new AccordRoutingKeyByteSource.VariableLength(ByteOrderedPartitioner.instance, ByteComparable.Version.OSS50)
+                                                                                                                                  : AccordRoutingKeyByteSource.create(key.asTokenKey().token().getPartitioner());
+
+            var read = serializer.fromComparableBytes(ByteArrayAccessor.instance, serializer.serialize(key));
+            Assertions.assertThat(read).isEqualTo(key);
+        });
+    }
+}

test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java

@@ -356,4 +356,4 @@ public class ProgressBarrierTest extends CMSTestBase
                              gen4.inflate(PCGFastPure.next(l, 2)));
         };
     }
-}
+}

test/unit/org/apache/cassandra/utils/RangeTreeTest.java

@@ -0,0 +1,568 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongUnaryOperator;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.impl.IntKey;
+import accord.impl.IntKey.Routing;
+import accord.primitives.Range;
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.RandomSource;
+import accord.utils.SearchableRangeList;
+import org.agrona.collections.IntArrayList;
+import org.agrona.collections.LongArrayList;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.qt;
+
+@RunWith(Parameterized.class)
+public class RangeTreeTest
+{
+    private static final Logger logger = LoggerFactory.getLogger(RangeTreeTest.class);
+    private static final Comparator<Routing> COMPARATOR = Comparator.naturalOrder();
+    private static final RangeTree.Accessor<Routing, Range> END_INCLUSIVE = new RangeTree.Accessor<>()
+    {
+        @Override
+        public Routing start(Range range)
+        {
+            return (Routing) range.start();
+        }
+
+        @Override
+        public Routing end(Range range)
+        {
+            return (Routing) range.end();
+        }
+
+        @Override
+        public boolean contains(Range range, Routing routing)
+        {
+            return range.contains(routing);
+        }
+
+        @Override
+        public boolean contains(Routing start, Routing end, Routing routing)
+        {
+            if (routing.compareTo(start) <= 0)
+                return false;
+            if (routing.compareTo(end) > 0)
+                return false;
+            return true;
+        }
+
+        @Override
+        public boolean intersects(Range range, Routing start, Routing end)
+        {
+            return range.compareIntersecting(IntKey.range(start, end)) == 0;
+        }
+
+        @Override
+        public boolean intersects(Range left, Range right)
+        {
+            return left.compareIntersecting(right) == 0;
+        }
+    };
+    private static final RangeTree.Accessor<Routing, Range> ALL_INCLUSIVE = new RangeTree.Accessor<>()
+    {
+        @Override
+        public Routing start(Range range)
+        {
+            return (Routing) range.start();
+        }
+
+        @Override
+        public Routing end(Range range)
+        {
+            return (Routing) range.end();
+        }
+
+        @Override
+        public boolean contains(Range range, Routing routing)
+        {
+            return range.contains(routing) || range.start().equals(routing);
+        }
+
+        @Override
+        public boolean contains(Routing start, Routing end, Routing routing)
+        {
+            if (routing.compareTo(start) < 0)
+                return false;
+            if (routing.compareTo(end) > 0)
+                return false;
+            return true;
+        }
+
+        @Override
+        public boolean intersects(Range range, Routing start, Routing end)
+        {
+            return range.compareIntersecting(IntKey.range(start, end)) == 0 || range.end().equals(start) || range.start().equals(end);
+        }
+
+        @Override
+        public boolean intersects(Range left, Range right)
+        {
+            return left.compareIntersecting(right) == 0 || left.end().equals(right.start()) || left.start().equals(right.end());
+        }
+    };
+
+    private static final Gen.IntGen SMALL_INT_GEN = rs -> rs.nextInt(0, 10);
+    private static final int MIN_TOKEN = 0, MAX_TOKEN = 1 << 16;
+    private static final int TOKEN_RANGE_SIZE = MAX_TOKEN - MIN_TOKEN + 1;
+    private static final Gen<Gen.IntGen> TOKEN_DISTRIBUTION = Gens.mixedDistribution(MIN_TOKEN, MAX_TOKEN + 1);
+    private static final Gen<Gen.IntGen> RANGE_SIZE_DISTRIBUTION = Gens.mixedDistribution(10, (int) (TOKEN_RANGE_SIZE * .01));
+
+    // Used to test different worse case patterns and see how the tree performs.
+    private enum Pattern
+    {
+        RANDOM, // tends to have high selectivity: matches 50-100% of the tree in testing
+        NO_OVERLP, // tests to have low selectivity; matches 1-2 elements in testing
+        SMALL_RANGES // lower selectivity than RANDOM but still matches ~30% of the tree in testing
+    }
+
+    // Having different models makes sure that the tree is flexiable enough and can be used with the semantics the user
+    // needs (with regard to inclusivity).  It also adds more confidence that the search logic is correct as different
+    // algorithems help validate this.
+    private enum ModelType {List, IntervalTree, SearchableRangeList}
+    private final Pattern pattern;
+    private final ModelType modelType;
+
+    public RangeTreeTest(Pattern pattern, ModelType modelType)
+    {
+        this.pattern = pattern;
+        this.modelType = modelType;
+    }
+
+    @Parameterized.Parameters(name = "{0}, {1}")
+    public static Collection<Object[]> data() {
+        return Stream.of(Pattern.values())
+                     .flatMap(p ->
+                              Stream.of(ModelType.values())
+                                    .map(m -> new Object[]{ p, m }))
+                     .collect(Collectors.toList());
+    }
+
+    @Test
+    public void test()
+    {
+        int samples = 3_000;
+        int examples = 10;
+        LongArrayList byToken = new LongArrayList(samples * examples, -1);
+        LongArrayList modelByToken = new LongArrayList(samples * examples, -1);
+        LongArrayList byTokenLength = new LongArrayList(samples * examples, -1);
+        LongArrayList byRange = new LongArrayList(samples * examples, -1);
+        LongArrayList modelByRange = new LongArrayList(samples * examples, -1);
+        LongArrayList byRangeLength = new LongArrayList(samples * examples, -1);
+        qt().withExamples(examples).check(rs -> {
+            var map = create(modelType);
+            var model = createModel(modelType);
+
+            Gen<Range> rangeGen = rangeGen(rs, pattern, samples);
+            for (int i = 0; i < samples; i++)
+            {
+                var range = rangeGen.next(rs);
+                var value = SMALL_INT_GEN.nextInt(rs);
+                map.put(range, value);
+                model.put(range, value);
+            }
+            model.done();
+            Assertions.assertThat(map.actual()).hasSize(samples);
+            if (rangeGen instanceof NoOverlap)
+                ((NoOverlap) rangeGen).reset();
+            Gen.IntGen tokenGe = TOKEN_DISTRIBUTION.next(rs);
+            for (int i = 0; i < samples; i++)
+            {
+                {
+                    // key lookup
+                    var lookup = IntKey.routing(tokenGe.nextInt(rs));
+                    var actual = timed(byToken, () -> map.intersectsToken(lookup));
+                    var expected = timed(modelByToken, () -> model.intersectsToken(lookup));
+                    byTokenLength.addLong(expected.size());
+                    Assertions.assertThat(sort(actual))
+                              .describedAs("Write=%d; token=%s", i, lookup)
+                              .isEqualTo(sort(expected));
+                }
+                {
+                    // range lookup
+                    var lookup = rangeGen.next(rs);
+                    var actual = timed(byRange, () -> map.intersects(lookup));
+                    var expected = timed(modelByRange, () -> model.intersects(lookup));
+                    byRangeLength.addLong(expected.size());
+                    Assertions.assertThat(sort(actual))
+                              .describedAs("Write=%d; range=%s", i, lookup)
+                              .isEqualTo(sort(expected));
+                }
+            }
+        });
+        StringBuilder sb = new StringBuilder();
+        sb.append("=======");
+        sb.append("\nPattern: " + pattern);
+        sb.append("\nModel: " + modelType);
+        sb.append("\nBy Token:");
+        sb.append("\n\tSizes: " + stats(byTokenLength, false));
+        sb.append("\n\t" + modelType + ": " + stats(modelByToken, true));
+        sb.append("\n\tTree: " + stats(byToken, true));
+        sb.append("\nBy Range:");
+        sb.append("\n\tSizes: " + stats(byRangeLength, false));
+        sb.append("\n\t" + modelType + ": " + stats(modelByRange, true));
+        sb.append("\n\tTree: " + stats(byRange, true));
+        logger.info(sb.toString());
+    }
+
+    private static class NoOverlap implements Gen<Range>
+    {
+        private final int delta;
+        private int idx = 0;
+
+        public NoOverlap(int samples)
+        {
+            this.delta = TOKEN_RANGE_SIZE / samples;
+        }
+
+        @Override
+        public Range next(RandomSource random)
+        {
+            int a = delta * idx++;
+            int b = a + delta;
+            return IntKey.range(a, b);
+        }
+
+        private void reset()
+        {
+            idx = 0;
+        }
+    }
+
+    private static Gen<Range> rangeGen(RandomSource randomSource, Pattern pattern, int samples)
+    {
+        Gen.IntGen tokenGen = TOKEN_DISTRIBUTION.next(randomSource);
+        switch (pattern)
+        {
+            case RANDOM:
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int b = tokenGen.nextInt(rs);
+                    while (a == b)
+                        b = tokenGen.nextInt(rs);
+                    if (a > b)
+                    {
+                        int tmp = a;
+                        a = b;
+                        b = tmp;
+                    }
+                    return IntKey.range(a, b);
+                };
+            case SMALL_RANGES:
+                Gen.IntGen rangeSizeGen = RANGE_SIZE_DISTRIBUTION.next(randomSource);
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int rangeSize = rangeSizeGen.nextInt(rs);
+                    int b = a + rangeSize;
+                    if (b > MAX_TOKEN)
+                    {
+                        b = a;
+                        a = b - rangeSize;
+                    }
+                    return IntKey.range(a, b);
+                };
+            case NO_OVERLP:
+                return new NoOverlap(samples);
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    private static String stats(LongArrayList list, boolean isTime)
+    {
+        LongUnaryOperator fn = isTime ? TimeUnit.NANOSECONDS::toMicros : l -> l;
+        String postfix = isTime ? "micro" : "";
+        long[] array = list.toLongArray();
+        Arrays.sort(array);
+        StringBuilder sb = new StringBuilder();
+        sb.append("Min: ").append(fn.applyAsLong(array[0])).append(postfix);
+        sb.append(", Median: ").append(fn.applyAsLong(array[array.length / 2])).append(postfix);
+        sb.append(", Max: ").append(fn.applyAsLong(array[array.length - 1])).append(postfix);
+        return sb.toString();
+    }
+
+    private static <T> T timed(LongArrayList target, Supplier<T> fn)
+    {
+        long nowNs = System.nanoTime();
+        try
+        {
+            return fn.get();
+        }
+        finally
+        {
+            target.add(System.nanoTime() - nowNs);
+        }
+    }
+
+    private static List<Map.Entry<Range, Integer>> sort(List<Map.Entry<Range, Integer>> array)
+    {
+        array.sort((a, b) -> {
+            int rc = a.getKey().compare(b.getKey());
+            if (rc == 0)
+                rc = a.getValue().compareTo(b.getValue());
+            return rc;
+        });
+        return array;
+    }
+
+    private interface Model
+    {
+        Object actual();
+
+        void put(Range range, int value);
+
+        List<Map.Entry<Range, Integer>> intersectsToken(Routing key);
+
+        List<Map.Entry<Range, Integer>> intersects(Range range);
+
+        void done();
+    }
+
+    private static RangeTreeModel create(ModelType modelType)
+    {
+        switch (modelType)
+        {
+            case List:
+            case SearchableRangeList:
+                return new RangeTreeModel(new RTree<>(COMPARATOR, END_INCLUSIVE));
+            case IntervalTree: return new RangeTreeModel(new RTree<>(COMPARATOR, ALL_INCLUSIVE));
+            default:
+                throw new AssertionError("Unknown type: " + modelType);
+        }
+    }
+
+    private static Model createModel(ModelType modelType)
+    {
+        switch (modelType)
+        {
+            case List: return new ListModel();
+            case SearchableRangeList: return new SearchableRangeListModel();
+            case IntervalTree: return new IntervalTreeModel();
+            default:
+                throw new AssertionError("Unknown type: " + modelType);
+        }
+    }
+
+    private static class RangeTreeModel implements Model
+    {
+        private final RangeTree<Routing, Range, Integer> tree;
+
+        private RangeTreeModel(RangeTree<Routing, Range, Integer> tree)
+        {
+            this.tree = tree;
+        }
+
+        @Override
+        public RangeTree<Routing, Range, Integer> actual()
+        {
+            return tree;
+        }
+
+        @Override
+        public void put(Range range, int value)
+        {
+            tree.add(range, value);
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersectsToken(Routing key)
+        {
+            return tree.searchToken(key);
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersects(Range range)
+        {
+            return tree.search(range);
+        }
+
+        @Override
+        public void done()
+        {
+
+        }
+    }
+
+    private static class ListModel implements Model
+    {
+        List<Map.Entry<Range, Integer>> actual = new ArrayList<>();
+
+        @Override
+        public List<Map.Entry<Range, Integer>> actual()
+        {
+            return actual;
+        }
+
+        @Override
+        public void put(Range range, int value)
+        {
+            actual.add(Map.entry(range, value));
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersectsToken(Routing key)
+        {
+            return actual.stream()
+                         .filter(p -> p.getKey().contains(key))
+                         .collect(Collectors.toList());
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersects(Range range)
+        {
+            return actual.stream()
+                         .filter(p -> p.getKey().compareIntersecting(range) == 0)
+                         .collect(Collectors.toList());
+        }
+
+        @Override
+        public void done()
+        {
+
+        }
+    }
+
+    private static class IntervalTreeModel implements Model
+    {
+        IntervalTree.Builder<Routing, Integer, Interval<Routing, Integer>> builder = IntervalTree.builder();
+        IntervalTree<Routing, Integer, Interval<Routing, Integer>> actual = null;
+
+        @Override
+        public IntervalTree<Routing, Integer, Interval<Routing, Integer>> actual()
+        {
+            return actual;
+        }
+
+        @Override
+        public void put(Range range, int value)
+        {
+            builder.add(new Interval<>((Routing) range.start(), (Routing) range.end(), value));
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersectsToken(Routing key)
+        {
+            return map(actual.matches(key));
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersects(Range range)
+        {
+            return map(actual.matches(new Interval<>((Routing) range.start(), (Routing) range.end(), null)));
+        }
+
+        private static List<Map.Entry<Range, Integer>> map(List<Interval<Routing, Integer>> matches)
+        {
+            return matches.stream().map(i -> Map.entry(IntKey.range(i.min, i.max), i.data)).collect(Collectors.toList());
+        }
+
+        @Override
+        public void done()
+        {
+            assert builder != null;
+            actual = builder.build();
+            builder = null;
+        }
+    }
+
+    private static class SearchableRangeListModel implements Model
+    {
+        private final Map<Range, IntArrayList> map = new HashMap<>();
+        private Range[] ranges;
+        private SearchableRangeList list = null;
+
+        @Override
+        public Object actual()
+        {
+            return list;
+        }
+
+        @Override
+        public void put(Range range, int value)
+        {
+            map.computeIfAbsent(range, ignore -> new IntArrayList()).addInt(value);
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersectsToken(Routing key)
+        {
+            List<Map.Entry<Range, Integer>> matches = new ArrayList<>();
+            // find ranges, then add the values
+            list.forEach(key, (a, b, c, d, idx) -> {
+                Range match = ranges[idx];
+                map.get(match).forEachInt(v -> matches.add(Map.entry(match, v)));
+            }, (a, b, c, d, start, end) -> {
+                for (int i = start; i < end; i++)
+                {
+                    Range match = ranges[i];
+                    map.get(match).forEachInt(v -> matches.add(Map.entry(match, v)));
+                }
+            }, 0, 0, 0, 0, 0);
+            return matches;
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> intersects(Range range)
+        {
+            List<Map.Entry<Range, Integer>> matches = new ArrayList<>();
+            // find ranges, then add the values
+            list.forEach(range, (a, b, c, d, idx) -> {
+                Range match = ranges[idx];
+                map.get(match).forEachInt(v -> matches.add(Map.entry(match, v)));
+            }, (a, b, c, d, start, end) -> {
+                for (int i = start; i < end; i++)
+                {
+                    Range match = ranges[i];
+                    map.get(match).forEachInt(v -> matches.add(Map.entry(match, v)));
+                }
+            }, 0, 0, 0, 0, 0);
+            return matches;
+        }
+
+        @Override
+        public void done()
+        {
+            List<Range> ranges = new ArrayList<>(map.keySet());
+            ranges.sort(Range::compare);
+            list = SearchableRangeList.build(this.ranges = ranges.toArray(Range[]::new));
+        }
+    }
+}

test/unit/org/apache/cassandra/utils/StatefulRangeTreeTest.java

@@ -0,0 +1,509 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import accord.api.RoutingKey;
+import accord.impl.IntKey;
+import accord.primitives.Range;
+import accord.utils.Gen;
+import accord.utils.Gens;
+import accord.utils.Property.Command;
+import accord.utils.Property.Commands;
+import accord.utils.Property.UnitCommand;
+import accord.utils.RandomSource;
+import org.apache.cassandra.service.accord.RangeTreeRangeAccessor;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.Property.stateful;
+
+public class StatefulRangeTreeTest
+{
+    private static final Gen.IntGen SMALL_INT_GEN = rs -> rs.nextInt(0, 10);
+    private static final Gen.IntGen NUM_CHILDREN_GEN = rs -> rs.nextInt(2, 12);
+    private static final Gen<Gen.IntGen> SIZE_TARGET_DISTRIBUTION = Gens.mixedDistribution(1 << 3, 1 << 9);
+    private static final int MIN_TOKEN = 0, MAX_TOKEN = 1 << 16;
+    private static final int TOKEN_RANGE_SIZE = MAX_TOKEN - MIN_TOKEN + 1;
+    private static final Gen<Gen.IntGen> TOKEN_DISTRIBUTION = Gens.mixedDistribution(MIN_TOKEN, MAX_TOKEN + 1);
+    private static final Gen<Gen.IntGen> RANGE_SIZE_DISTRIBUTION = Gens.mixedDistribution(10, (int) (TOKEN_RANGE_SIZE * .01));
+    static final Comparator<Map.Entry<Range, Integer>> COMPARATOR = (a, b) -> {
+        int rc = a.getKey().compare(b.getKey());
+        if (rc == 0)
+            rc = a.getValue().compareTo(b.getValue());
+        return rc;
+    };
+
+    /**
+     * Stateful test for RTree.
+     *
+     * This test is very similar to {@link RangeTreeTest#test} but is fully mutable, so can not
+     * use the immutable search trees (else rebuidling becomes a large cost).  Both tests should exist as they use different
+     * models, which helps build confidence that the RTree does the correct thing; that test also covers start and end
+     * inclusive, which this test does not.
+     */
+    @Test
+    public void test()
+    {
+        stateful().check(new Commands<State, Sut>()
+        {
+            @Override
+            public Gen<State> genInitialState()
+            {
+                return rs -> {
+                    Gen<Range> rangeGen = rangeGen(rs);
+                    int numChildren = NUM_CHILDREN_GEN.nextInt(rs);
+                    int sizeTarget = SIZE_TARGET_DISTRIBUTION.next(rs).filter(s -> s > numChildren).nextInt(rs);
+                    int createWeight = rs.nextInt(1, 100);
+                    int updateWeight = rs.nextInt(1, 20);
+                    int deleteWeight = rs.nextInt(1, 20);
+                    int clearWeight = rs.nextInt(0, 2); // either disabled or enabled with weight=1
+                    int readWeight = rs.nextInt(1, 20);
+                    return new State(sizeTarget, numChildren,
+                                     TOKEN_DISTRIBUTION.next(rs), rangeGen,
+                                     createWeight, updateWeight, deleteWeight, clearWeight, readWeight);
+                };
+            }
+
+            @Override
+            public Sut createSut(State state)
+            {
+                return new Sut(state.sizeTarget, state.numChildren);
+            }
+
+            @Override
+            public Gen<Command<State, Sut, ?>> commands(State state)
+            {
+                Map<Gen<Command<State, Sut, ?>>, Integer> possible = new HashMap<>();
+                possible.put(rs -> new Create(state.newRange(rs), SMALL_INT_GEN.nextInt(rs)), state.createWeight);
+                possible.put(rs -> new Read(state.newRange(rs)), state.readWeight);
+                possible.put(rs -> new KeyRead(IntKey.routing(state.tokenGen.nextInt(rs))), state.readWeight);
+                possible.put(rs -> new RangeRead(state.rangeGen.next(rs)), state.readWeight);
+                possible.put(ignore -> Iterate.instance, state.readWeight);
+                possible.put(ignore -> Clear.instance, state.clearWeight);
+                if (!state.uniqRanges.isEmpty())
+                {
+                    possible.put(rs -> new Read(rs.pick(state.uniqRanges)), state.readWeight);
+                    possible.put(rs -> {
+                        Range range = rs.pick(state.uniqRanges);
+                        int token = rs.nextInt(((IntKey.Routing) range.start()).key, ((IntKey.Routing) range.end()).key) + 1;
+                        return new KeyRead(IntKey.routing(token));
+                    }, state.readWeight);
+                    possible.put(rs -> new RangeRead(rs.pick(state.uniqRanges)), state.readWeight);
+                    possible.put(rs -> new Update(rs.pick(state.uniqRanges), SMALL_INT_GEN.nextInt(rs)), state.updateWeight);
+                    possible.put(rs -> new Delete(rs.pick(state.uniqRanges)), state.deleteWeight);
+                }
+                return Gens.oneOf(possible);
+            }
+        });
+    }
+
+    private static Gen<Range> rangeGen(RandomSource rand)
+    {
+        Gen.IntGen tokenGen = TOKEN_DISTRIBUTION.next(rand);
+        switch (rand.nextInt(0, 3))
+        {
+            case 0: // pure random
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int b = tokenGen.nextInt(rs);
+                    while (a == b)
+                        b = tokenGen.nextInt(rs);
+                    if (a > b)
+                    {
+                        int tmp = a;
+                        a = b;
+                        b = tmp;
+                    }
+                    return IntKey.range(a, b);
+                };
+            case 1: // small range
+                Gen.IntGen rangeSizeGen = RANGE_SIZE_DISTRIBUTION.next(rand);
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int rangeSize = rangeSizeGen.nextInt(rs);
+                    int b = a + rangeSize;
+                    if (b > MAX_TOKEN)
+                    {
+                        b = a;
+                        a = b - rangeSize;
+                    }
+                    return IntKey.range(a, b);
+                };
+            case 2: // single element
+                return rs -> {
+                    int a = tokenGen.nextInt(rs);
+                    int b = a + 1;
+                    return IntKey.range(a, b);
+                };
+            default:
+                throw new AssertionError();
+        }
+    }
+
+    static class Create implements UnitCommand<State, Sut>
+    {
+        private final Range range;
+        private final int value;
+
+        Create(Range range, int value)
+        {
+            this.range = range;
+            this.value = value;
+        }
+
+        @Override
+        public void applyUnit(State state)
+        {
+            state.add(range, value);
+        }
+
+        @Override
+        public void runUnit(Sut sut)
+        {
+            sut.tree.add(range, value);
+        }
+
+        @Override
+        public void checkPostconditions(State state, Void expected,
+                                        Sut sut, Void actual)
+        {
+            Assertions.assertThat(sut.tree.size()).isEqualTo(state.list.size());
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Create(" + range + ", " + value + ")";
+        }
+    }
+
+    static abstract class AbstractRead<T> implements Command<State, Sut, List<T>>
+    {
+        private final Comparator<T> comparator;
+
+        protected AbstractRead(Comparator<T> comparator)
+        {
+            this.comparator = comparator;
+        }
+
+        @Override
+        public void checkPostconditions(State state, List<T> expected,
+                                        Sut sut, List<T> actual)
+        {
+            expected.sort(comparator);
+            actual.sort(comparator);
+            Assertions.assertThat(actual).isEqualTo(expected);
+        }
+    }
+
+    static class Read extends AbstractRead<Integer>
+    {
+        private final Range range;
+
+        Read(Range range)
+        {
+            super(Comparator.naturalOrder());
+            this.range = range;
+        }
+
+        @Override
+        public List<Integer> apply(State state)
+        {
+            return state.get(range);
+        }
+
+        @Override
+        public List<Integer> run(Sut sut)
+        {
+            return sut.tree.get(range);
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Read(" + range + ")";
+        }
+    }
+
+    static class RangeRead extends AbstractRead<Map.Entry<Range, Integer>>
+    {
+        private final Range range;
+
+        RangeRead(Range range)
+        {
+            super(COMPARATOR);
+            this.range = range;
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> apply(State state)
+        {
+            return state.list.stream().filter(e -> e.getKey().compareIntersecting(range) == 0).collect(Collectors.toList());
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> run(Sut sut)
+        {
+            return sut.tree.search(range);
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Range Read(" + range + ")";
+        }
+    }
+
+    static class KeyRead extends AbstractRead<Map.Entry<Range, Integer>>
+    {
+        final RoutingKey key;
+
+        KeyRead(RoutingKey key)
+        {
+            super(COMPARATOR);
+            this.key = key;
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> apply(State state)
+        {
+            return state.list.stream().filter(e -> e.getKey().contains(key)).collect(Collectors.toList());
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> run(Sut sut)
+        {
+            return sut.tree.searchToken(key);
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Token Read(" + key + ")";
+        }
+    }
+
+    static class Update implements UnitCommand<State, Sut>
+    {
+        private final Range range;
+        private final int value;
+
+        Update(Range range, int value)
+        {
+            this.range = range;
+            this.value = value;
+        }
+
+        @Override
+        public void applyUnit(State state)
+        {
+            state.update(range, value);
+        }
+
+        @Override
+        public void runUnit(Sut sut)
+        {
+            sut.tree.get(range, e -> e.setValue(value));
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Update(" + range + ", " + value + ")";
+        }
+    }
+
+    static class Delete implements UnitCommand<State, Sut>
+    {
+        private final Range range;
+
+        Delete(Range range)
+        {
+            this.range = range;
+        }
+
+        @Override
+        public void applyUnit(State state)
+        {
+            state.remove(range);
+        }
+
+        @Override
+        public void runUnit(Sut sut)
+        {
+            sut.tree.remove(range);
+        }
+
+        @Override
+        public void checkPostconditions(State state, Void expected,
+                                        Sut sut, Void actual)
+        {
+            Assertions.assertThat(sut.tree.size()).isEqualTo(state.list.size());
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Delete(" + range + ")";
+        }
+    }
+
+    static class Clear implements UnitCommand<State, Sut>
+    {
+        static final Clear instance = new Clear();
+
+        @Override
+        public void applyUnit(State state)
+        {
+            state.uniqRanges.clear();
+            state.list.clear();
+        }
+
+        @Override
+        public void runUnit(Sut sut)
+        {
+            sut.tree.clear();
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Clear(size=" + state.list.size() + ")";
+        }
+    }
+
+    static class Iterate extends AbstractRead<Map.Entry<Range, Integer>>
+    {
+        static final Iterate instance = new Iterate();
+        public Iterate()
+        {
+            super(COMPARATOR);
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> apply(State state)
+        {
+            return state.list;
+        }
+
+        @Override
+        public List<Map.Entry<Range, Integer>> run(Sut sut)
+        {
+            return sut.tree.stream().collect(Collectors.toList());
+        }
+
+        @Override
+        public String detailed(State state)
+        {
+            return "Iterate(size=" + state.list.size() + ")";
+        }
+    }
+
+    private static class State
+    {
+        private final List<Map.Entry<Range, Integer>> list = new ArrayList<>();
+        private final TreeSet<Range> uniqRanges = new TreeSet<>(Range::compare);
+        private final int sizeTarget, numChildren;
+        private final Gen.IntGen tokenGen;
+        private final Gen<Range> rangeGen;
+        private final int createWeight, updateWeight, deleteWeight, clearWeight, readWeight;
+
+        private State(int sizeTarget, int numChildren,
+                      Gen.IntGen tokenGen, Gen<Range> rangeGen,
+                      int createWeight, int updateWeight, int deleteWeight, int clearWeight, int readWeight)
+        {
+            this.sizeTarget = sizeTarget;
+            this.numChildren = numChildren;
+            this.tokenGen = tokenGen;
+            this.rangeGen = rangeGen;
+            this.createWeight = createWeight;
+            this.updateWeight = updateWeight;
+            this.deleteWeight = deleteWeight;
+            this.clearWeight = clearWeight;
+            this.readWeight = readWeight;
+        }
+
+        public Range newRange(RandomSource rs)
+        {
+            Range range;
+            while ((uniqRanges.contains(range = rangeGen.next(rs)))) {}
+            return range;
+        }
+
+        public void add(Range range, int value)
+        {
+            list.add(new MutableEntry<>(range, value));
+            uniqRanges.add(range);
+        }
+
+        public List<Integer> get(Range range)
+        {
+            if (!uniqRanges.contains(range))
+                return Collections.emptyList();
+            return list.stream().filter(e -> e.getKey().equals(range)).map(e -> e.getValue()).collect(Collectors.toList());
+        }
+
+        public void update(Range range, int value)
+        {
+            if (!uniqRanges.contains(range))
+                return;
+            list.forEach(e -> {
+                if (e.getKey().equals(range))
+                    e.setValue(value);
+            });
+        }
+
+        public void remove(Range range)
+        {
+            if (!uniqRanges.contains(range))
+                return;
+            uniqRanges.remove(range);
+            list.removeIf(e -> e.getKey().equals(range));
+        }
+
+        @Override
+        public String toString()
+        {
+            return "State{" +
+                   "sizeTarget=" + sizeTarget +
+                   ", numChildren=" + numChildren +
+                   '}';
+        }
+    }
+
+    public static class Sut
+    {
+        private final RangeTree<RoutingKey, Range, Integer> tree;
+
+        private Sut(int sizeTarget, int numChildren)
+        {
+            tree = new RTree(Comparator.naturalOrder(), RangeTreeRangeAccessor.instance, sizeTarget, numChildren);
+        }
+    }
+}