Live migration for non-serial reads

Patch by Ariel Weisberg; Reviewed by David Capwell for CASSANDRA-19439

build.xml

@@ -1239,6 +1239,7 @@
         <jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" />
         <jvmarg value="-XX:ActiveProcessorCount=${cassandra.test.processorCount}" />
         <jvmarg value="-XX:HeapDumpPath=${build.test.dir}" />
+        <jvmarg value="-Dcassandra.test.accord.allow_test_modes=true"/>
         <jvmarg value="-Dcassandra.test.driver.connection_timeout_ms=${test.driver.connection_timeout_ms}"/>
         <jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/>
         <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>

ide/idea/workspace.xml

@@ -190,6 +190,7 @@
                                           -Dcassandra.ring_delay_ms=10000
                                           -Dcassandra.skip_sync=true
                                           -Dcassandra.storagedir=$PROJECT_DIR$/data
+                                          -Dcassandra.test.accord.allow_test_modes=true
                                           -Dcassandra.strict.runtime.checks=true
                                           -Dcassandra.test.flush_local_schema_changes=false
                                           -Dcassandra.test.messagingService.nonGracefulShutdown=true

src/java/org/apache/cassandra/batchlog/BatchlogManager.java

@@ -31,7 +31,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
-
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -454,12 +453,12 @@ public class BatchlogManager implements BatchlogManagerMBean
                 if (accordResult != null)
                 {
                     IAccordService accord = AccordService.instance();
-                    TxnResult.Kind kind = accord.getTxnResult(accordResult, true, ConsistencyLevel.QUORUM, accordTxnStart).kind();
+                    TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
                     if (kind == retry_new_protocol)
                         throw new RetryOnDifferentSystemException();
                 }
             }
-            catch (WriteTimeoutException|WriteFailureException|RetryOnDifferentSystemException  e)
+            catch (WriteTimeoutException | WriteFailureException | RetryOnDifferentSystemException e )
             {
                 logger.trace("Failed replaying a batched mutation on Accord, will write a hint");
                 logger.trace("Failure was : {}", e.getMessage());

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -39,6 +39,7 @@ import static org.apache.cassandra.utils.LocalizeString.toUpperCaseLocalized;
 public enum CassandraRelevantProperties
 {
     ACCORD_AGENT_CLASS("cassandra.test.accord.agent"),
+    ACCORD_ALLOW_TEST_MODES("cassandra.test.accord.allow_test_modes", "false"),
     ACCORD_KEY_PARANOIA_COSTFACTOR(Invariants.KEY_PARANOIA_COSTFACTOR),
     ACCORD_KEY_PARANOIA_CPU(Invariants.KEY_PARANOIA_CPU),
     ACCORD_KEY_PARANOIA_MEMORY(Invariants.KEY_PARANOIA_MEMORY),

src/java/org/apache/cassandra/cql3/statements/BatchStatement.java

@@ -53,6 +53,7 @@ import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.Slice;
 import org.apache.cassandra.db.Slices;
@@ -356,7 +357,7 @@ public class BatchStatement implements CQLStatement.CompositeCQLStatement
         }
         // local is either executeWithoutConditions modifying a virtual table (doesn't support txns) or executeLocal
         // which is called by test or internal things that are bypassing distributed system modification/checks
-        return collector.toMutations(state, local);
+        return collector.toMutations(state, local ? PotentialTxnConflicts.ALLOW : PotentialTxnConflicts.DISALLOW);
     }
 
     /**

src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.db.CounterMutation;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogSegment;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -137,14 +138,14 @@ final class BatchUpdatesCollector implements UpdatesCollector
      * @return a collection containing all the mutations.
      */
     @Override
-    public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
+    public List<IMutation> toMutations(ClientState state, PotentialTxnConflicts potentialTxnConflicts)
     {
         List<IMutation> ms = new ArrayList<>();
         for (Map<ByteBuffer, IMutationBuilder> ksMap : mutationBuilders.values())
         {
             for (IMutationBuilder builder : ksMap.values())
             {
-                IMutation mutation = builder.build(allowPotentialTxnConflicts);
+                IMutation mutation = builder.build(potentialTxnConflicts);
                 mutation.validateIndexedColumns(state);
                 mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);
                 ms.add(mutation);
@@ -182,7 +183,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
         /**
          * Build the immutable mutation
          */
-        IMutation build(boolean allowPotentialTxnConflicts);
+        IMutation build(PotentialTxnConflicts potentialTxnConflicts);
 
         /**
          * Get the builder for the given tableId
@@ -215,7 +216,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
             return this;
         }
 
-        public Mutation build(boolean allowPotentialTxnConflicts)
+        public Mutation build(PotentialTxnConflicts potentialTxnConflicts)
         {
             ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
             for (Map.Entry<TableId, PartitionUpdate.Builder> updateEntry : modifications.entrySet())
@@ -223,7 +224,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
                 PartitionUpdate update = updateEntry.getValue().build();
                 updates.put(updateEntry.getKey(), update);
             }
-            return new Mutation(keyspaceName, key, updates.build(), createdAt, allowPotentialTxnConflicts);
+            return new Mutation(keyspaceName, key, updates.build(), createdAt, potentialTxnConflicts);
         }
 
         public PartitionUpdate.Builder get(TableId tableId)
@@ -263,9 +264,9 @@ final class BatchUpdatesCollector implements UpdatesCollector
             return mutationBuilder.add(builder);
         }
 
-        public IMutation build(boolean allowPotentialTxnConflicts)
+        public IMutation build(PotentialTxnConflicts potentialTxnConflicts)
         {
-            return new CounterMutation(mutationBuilder.build(allowPotentialTxnConflicts), cl);
+            return new CounterMutation(mutationBuilder.build(potentialTxnConflicts), cl);
         }
 
         public PartitionUpdate.Builder get(TableId id)
@@ -297,7 +298,7 @@ final class BatchUpdatesCollector implements UpdatesCollector
         }
 
         @Override
-        public VirtualMutation build(boolean allowPotentialTxnConflicts)
+        public VirtualMutation build(PotentialTxnConflicts potentialTxnConflicts)
         {
             ImmutableMap.Builder<TableId, PartitionUpdate> updates = new ImmutableMap.Builder<>();
             modifications.forEach((tableId, updateBuilder) -> updates.put(tableId, updateBuilder.build()));

src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java

@@ -63,8 +63,8 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.accord.txn.TxnCondition;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnReference;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.service.accord.txn.TxnUpdate;
@@ -83,7 +83,6 @@ import static org.apache.cassandra.service.accord.txn.TxnData.txnDataName;
 import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
 
-
 /**
  * Processed CAS conditions and update on potentially multiple rows of the same partition.
  */
@@ -504,8 +503,8 @@ public class CQL3CasRequest implements CASRequest
         // If the write strategy is sending all writes through Accord there is no need to use the supplied consistency
         // level since Accord will manage reading safely
         TableParams tableParams = getTableMetadata(cm, metadata.id).params;
-        consistencyLevel = tableParams.transactionalMode.readCLForStrategy(tableParams.transactionalMigrationFrom, consistencyLevel, cm, metadata.id, readCommand.partitionKey().getToken());
-        TxnKeyRead read = TxnKeyRead.createCasRead(readCommand, consistencyLevel);
+        consistencyLevel = tableParams.transactionalMode.readCLForMode(tableParams.transactionalMigrationFrom, consistencyLevel, cm, metadata.id, readCommand.partitionKey().getToken());
+        TxnRead read = TxnRead.createCasRead(readCommand, consistencyLevel);
         // In a CAS requesting only one key is supported and writes
         // can't be dependent on any data that is read (only conditions)
         // so the only relevant keys are the read key
@@ -518,7 +517,7 @@ public class CQL3CasRequest implements CASRequest
         // since it is safe to match what non-SERIAL writes do
         TableMetadata tableMetadata = getTableMetadata(cm, metadata.id);
         TableParams tableParams = tableMetadata.params;
-        commitConsistencyLevel = tableParams.transactionalMode.commitCLForStrategy(tableParams.transactionalMigrationFrom, commitConsistencyLevel, cm, metadata.id, key.getToken());
+        commitConsistencyLevel = tableParams.transactionalMode.commitCLForMode(tableParams.transactionalMigrationFrom, commitConsistencyLevel, cm, metadata.id, key.getToken());
         // CAS requires using the new txn timestamp to correctly linearize some kinds of updates
         return new TxnUpdate(createWriteFragments(clientState), createCondition(), commitConsistencyLevel, false);
     }

src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java

@@ -73,6 +73,7 @@ import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
@@ -870,7 +871,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             SingleTableSinglePartitionUpdatesCollector collector = new SingleTableSinglePartitionUpdatesCollector(metadata, updatedColumns);
             addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate);
             // local means this is test or internal things that are bypassing distributed system modification/checks
-            return collector.toMutations(state, local);
+            return collector.toMutations(state, local ? PotentialTxnConflicts.ALLOW : PotentialTxnConflicts.DISALLOW);
         }
         else
         {
@@ -878,7 +879,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             SingleTableUpdatesCollector collector = new SingleTableUpdatesCollector(metadata, updatedColumns, perPartitionKeyCounts);
             addUpdates(collector, keys, state, options, local, timestamp, nowInSeconds, requestTime, constructingAccordBaseUpdate);
             // local means this is test or internal things that are bypassing distributed system modification/checks
-            return collector.toMutations(state, local);
+            return collector.toMutations(state, local ? PotentialTxnConflicts.ALLOW : PotentialTxnConflicts.DISALLOW);
         }
     }
 

src/java/org/apache/cassandra/cql3/statements/SelectStatement.java

@@ -79,6 +79,7 @@ import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.PartitionRangeReadQuery;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.ReadQuery;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
@@ -386,7 +387,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
             }
         }
 
-        ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(), nowInSec, limit);
+        ReadQuery query = getQuery(options, state.getClientState(), selectors.getColumnFilter(), nowInSec, limit, PotentialTxnConflicts.DISALLOW);
 
         if (options.isReadThresholdsEnabled())
             query.trackWarnings();
@@ -437,7 +438,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
                         getLimit(options),
                         getPerPartitionLimit(options),
                         options.getPageSize(),
-                        getAggregationSpec(options));
+                        getAggregationSpec(options),
+                        PotentialTxnConflicts.DISALLOW);
     }
 
     public ReadQuery getQuery(QueryOptions options,
@@ -447,18 +449,20 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
                               int userLimit,
                               int perPartitionLimit,
                               int pageSize,
-                              AggregationSpecification aggregationSpec)
+                              AggregationSpecification aggregationSpec,
+                              PotentialTxnConflicts potentialTxnConflicts)
     {
         DataLimits limit = getDataLimits(userLimit, perPartitionLimit, pageSize, aggregationSpec);
 
-        return getQuery(options, state, columnFilter, nowInSec, limit);
+        return getQuery(options, state, columnFilter, nowInSec, limit, potentialTxnConflicts);
     }
 
     public ReadQuery getQuery(QueryOptions options,
                               ClientState state,
                               ColumnFilter columnFilter,
                               long nowInSec,
-                              DataLimits limit)
+                              DataLimits limit,
+                              PotentialTxnConflicts potentialTxnConflicts)
     {
         RowFilter rowFilter = getRowFilter(options, state);
 
@@ -467,13 +471,13 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
             if (restrictions.usesSecondaryIndexing() && !SchemaConstants.isLocalSystemKeyspace(table.keyspace))
                 Guardrails.nonPartitionRestrictedIndexQueryEnabled.ensureEnabled(state);
 
-            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec);
+            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec, potentialTxnConflicts);
         }
 
         if (restrictions.usesSecondaryIndexing() && !rowFilter.isStrict())
-            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec);
+            return getRangeCommand(options, state, columnFilter, rowFilter, limit, nowInSec, potentialTxnConflicts);
 
-        return getSliceCommands(options, state, columnFilter, rowFilter, limit, nowInSec);
+        return getSliceCommands(options, state, columnFilter, rowFilter, limit, nowInSec, potentialTxnConflicts);
     }
 
     private ResultMessage.Rows execute(ReadQuery query,
@@ -657,7 +661,8 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
                                    userLimit,
                                    userPerPartitionLimit,
                                    pageSize,
-                                   aggregationSpec);
+                                   aggregationSpec,
+                                   PotentialTxnConflicts.ALLOW);
 
         try (ReadExecutionController executionController = query.executionController())
         {
@@ -704,7 +709,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
             throw new IllegalStateException();
 
         Selectors selectors = selection.newSelectors(options);
-        ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE, null);
+        ReadQuery query = getQuery(options, state, selectors.getColumnFilter(), nowInSec, userLimit, userPerPartitionLimit, Integer.MAX_VALUE, null, PotentialTxnConflicts.ALLOW);
 
         Map<DecoratedKey, List<Row>> result = Collections.emptyMap();
         try (ReadExecutionController executionController = query.executionController())
@@ -778,7 +783,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
     }
 
     private ReadQuery getSliceCommands(QueryOptions options, ClientState state, ColumnFilter columnFilter,
-                                       RowFilter rowFilter, DataLimits limit, long nowInSec)
+                                       RowFilter rowFilter, DataLimits limit, long nowInSec, PotentialTxnConflicts potentialTxnConflicts)
     {
         Collection<ByteBuffer> keys = restrictions.getPartitionKeys(options, state);
         if (keys.isEmpty())
@@ -801,7 +806,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
         }
 
         SinglePartitionReadQuery.Group<? extends SinglePartitionReadQuery> group =
-            SinglePartitionReadQuery.createGroup(table, nowInSec, columnFilter, rowFilter, limit, decoratedKeys, filter);
+            SinglePartitionReadQuery.createGroup(table, nowInSec, columnFilter, rowFilter, limit, decoratedKeys, filter, potentialTxnConflicts);
 
         // If there's a secondary index that the commands can use, have it validate the request parameters.
         group.maybeValidateIndex();
@@ -855,7 +860,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
     }
 
     private ReadQuery getRangeCommand(QueryOptions options, ClientState state, ColumnFilter columnFilter,
-                                      RowFilter rowFilter, DataLimits limit, long nowInSec)
+                                      RowFilter rowFilter, DataLimits limit, long nowInSec, PotentialTxnConflicts potentialTxnConflicts)
     {
         ClusteringIndexFilter clusteringIndexFilter = makeClusteringIndexFilter(options, state, columnFilter);
         if (clusteringIndexFilter == null)
@@ -868,7 +873,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement,
             return ReadQuery.empty(table);
 
         ReadQuery command =
-            PartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter));
+            PartitionRangeReadQuery.create(table, nowInSec, columnFilter, rowFilter, limit, new DataRange(keyBounds, clusteringIndexFilter), potentialTxnConflicts);
 
         // If there's a secondary index that the command can use, have it validate the request parameters.
         command.maybeValidateIndex();

src/java/org/apache/cassandra/cql3/statements/SingleTableSinglePartitionUpdatesCollector.java

@@ -25,6 +25,7 @@ import org.apache.cassandra.db.CounterMutation;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogSegment;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -78,16 +79,16 @@ final class SingleTableSinglePartitionUpdatesCollector implements UpdatesCollect
      * Returns a collection containing all the mutations.
      */
     @Override
-    public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
+    public List<IMutation> toMutations(ClientState state, PotentialTxnConflicts potentialTxnConflicts)
     {
         // it is possible that a modification statement does not create any mutations
         // for example: DELETE FROM some_table WHERE part_key = 1 AND clust_key < 3 AND clust_key > 5
         if (builder == null)
             return Collections.emptyList();
-        return Collections.singletonList(createMutation(state, builder, allowPotentialTxnConflicts));
+        return Collections.singletonList(createMutation(state, builder, potentialTxnConflicts));
     }
 
-    private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, boolean allowPotentialTxnConflicts)
+    private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, PotentialTxnConflicts potentialTxnConflicts)
     {
         IMutation mutation;
 
@@ -96,7 +97,7 @@ final class SingleTableSinglePartitionUpdatesCollector implements UpdatesCollect
         else if (metadata.isCounter())
             mutation = new CounterMutation(new Mutation(builder.build()), counterConsistencyLevel);
         else
-            mutation = new Mutation(builder.build(), allowPotentialTxnConflicts);
+            mutation = new Mutation(builder.build(), potentialTxnConflicts);
 
         mutation.validateIndexedColumns(state);
         mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);

src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.db.CounterMutation;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogSegment;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -95,24 +96,24 @@ final class SingleTableUpdatesCollector implements UpdatesCollector
      * @return a collection containing all the mutations.
      */
     @Override
-    public List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts)
+    public List<IMutation> toMutations(ClientState state, PotentialTxnConflicts potentialTxnConflicts)
     {
         if (puBuilders.size() == 1)
         {
             PartitionUpdate.Builder builder = puBuilders.values().iterator().next();
-            return Collections.singletonList(createMutation(state, builder, allowPotentialTxnConflicts));
+            return Collections.singletonList(createMutation(state, builder, potentialTxnConflicts));
         }
         List<IMutation> ms = new ArrayList<>(puBuilders.size());
         for (PartitionUpdate.Builder builder : puBuilders.values())
         {
-            IMutation mutation = createMutation(state, builder, allowPotentialTxnConflicts);
+            IMutation mutation = createMutation(state, builder, potentialTxnConflicts);
             ms.add(mutation);
         }
 
         return ms;
     }
 
-    private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, boolean allowPotentialTxnConflicts)
+    private IMutation createMutation(ClientState state, PartitionUpdate.Builder builder, PotentialTxnConflicts potentialTxnConflicts)
     {
         IMutation mutation;
 
@@ -121,7 +122,7 @@ final class SingleTableUpdatesCollector implements UpdatesCollector
         else if (metadata.isCounter())
             mutation = new CounterMutation(new Mutation(builder.build()), counterConsistencyLevel);
         else
-            mutation = new Mutation(builder.build(), allowPotentialTxnConflicts);
+            mutation = new Mutation(builder.build(), potentialTxnConflicts);
 
         mutation.validateIndexedColumns(state);
         mutation.validateSize(MessagingService.current_version, CommitLogSegment.ENTRY_OVERHEAD_SIZE);

src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java

@@ -40,6 +40,7 @@ import com.google.common.collect.Iterables;
 
 import accord.api.Key;
 import accord.primitives.Keys;
+import accord.primitives.Routable.Domain;
 import accord.primitives.Txn;
 import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.audit.AuditLogContext;
@@ -56,34 +57,43 @@ import org.apache.cassandra.cql3.transactions.ConditionStatement;
 import org.apache.cassandra.cql3.transactions.ReferenceOperation;
 import org.apache.cassandra.cql3.transactions.RowDataReference;
 import org.apache.cassandra.cql3.transactions.SelectReferenceSource;
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.SinglePartitionReadQuery;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
 import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.accord.api.AccordRoutableKey;
+import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.service.accord.txn.TxnCondition;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
 import org.apache.cassandra.service.accord.txn.TxnNamedRead;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnReference;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.service.accord.txn.TxnUpdate;
 import org.apache.cassandra.service.accord.txn.TxnWrite;
+import org.apache.cassandra.service.consensus.TransactionalMode;
+import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static accord.primitives.Txn.Kind.Read;
+import static com.google.common.base.Preconditions.checkArgument;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
 import static org.apache.cassandra.cql3.statements.RequestValidations.checkTrue;
@@ -92,8 +102,9 @@ import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.AU
 import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.RETURNING;
 import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.USER;
 import static org.apache.cassandra.service.accord.txn.TxnData.txnDataName;
-import static org.apache.cassandra.service.accord.txn.TxnKeyRead.createTxnRead;
+import static org.apache.cassandra.service.accord.txn.TxnRead.createTxnRead;
 import static org.apache.cassandra.service.accord.txn.TxnResult.Kind.retry_new_protocol;
+import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.shouldReadEphemerally;
 
 public class TransactionStatement implements CQLStatement.CompositeCQLStatement, CQLStatement.ReturningCQLStatement
@@ -261,8 +272,8 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         for (NamedSelect select : assignments)
         {
             TxnNamedRead read = createNamedRead(select, options, state);
+            keyConsumer.accept((Key)read.key());
             minEpoch = Math.max(minEpoch, read.command().metadata().epoch.getEpoch());
-            keyConsumer.accept(read.key());
             reads.add(read);
         }
 
@@ -270,7 +281,7 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         {
             for (TxnNamedRead read : createNamedReads(returningSelect, options, state))
             {
-                keyConsumer.accept(read.key());
+                keyConsumer.accept((Key)read.key());
                 minEpoch = Math.max(minEpoch, read.command().metadata().epoch.getEpoch());
                 reads.add(read);
             }
@@ -279,8 +290,11 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         if (autoReads != null)
         {
             for (NamedSelect select : autoReads.values())
-                // don't need keyConsumer as the keys are known to exist due to Modification
-                reads.add(createNamedRead(select, options, state));
+            {
+                TxnNamedRead read = createNamedRead(select, options, state);
+                keyConsumer.accept((Key)read.key());
+                reads.add(read);
+            }
         }
 
         return reads;
@@ -301,15 +315,15 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         return new TxnCondition.BooleanGroup(TxnCondition.Kind.AND, result);
     }
 
-    List<TxnWrite.Fragment> createWriteFragments(ClientState state, QueryOptions options, Map<Integer, NamedSelect> autoReads, Consumer<Key> keyConsumer)
+    List<TxnWrite.Fragment> createWriteFragments(ClientState state, QueryOptions options, Map<Integer, NamedSelect> autoReads, Set<Key> keys)
     {
         List<TxnWrite.Fragment> fragments = new ArrayList<>(updates.size());
         int idx = 0;
         for (ModificationStatement modification : updates)
         {
             TxnWrite.Fragment fragment = modification.getTxnWriteFragment(idx, state, options);
+            keys.add(fragment.key);
             minEpoch = Math.max(minEpoch, fragment.baseUpdate.metadata().epoch.getEpoch());
-            keyConsumer.accept(fragment.key);
             fragments.add(fragment);
 
             if (modification.allReferenceOperations().stream().anyMatch(ReferenceOperation::requiresRead))
@@ -325,9 +339,12 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         return fragments;
     }
 
-    AccordUpdate createUpdate(ClientState state, QueryOptions options, Map<Integer, NamedSelect> autoReads, Consumer<Key> keyConsumer)
+    AccordUpdate createUpdate(ClusterMetadata cm, ClientState state, QueryOptions options, Map<Integer, NamedSelect> autoReads, Set<Key> keys)
     {
-        return new TxnUpdate(createWriteFragments(state, options, autoReads, keyConsumer), createCondition(options), null, false);
+        checkArgument(keys.isEmpty(), "Construct update before reads so the key set can be used to determine commit consistency level");
+        List<TxnWrite.Fragment> writeFragments = createWriteFragments(state, options, autoReads, keys);
+        ConsistencyLevel commitCL = consistencyLevelForAccordCommit(cm, keys, options.getConsistency());
+        return new TxnUpdate(writeFragments, createCondition(options), commitCL, false);
     }
 
     Keys toKeys(SortedSet<Key> keySet)
@@ -335,10 +352,90 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
         return new Keys(keySet);
     }
 
+    private ConsistencyLevel consistencyLevelForAccordRead(ClusterMetadata cm, Set<Key> keys, @Nullable ConsistencyLevel consistencyLevel)
+    {
+        // Write transactions are read/write so it creates a read and ends up needing a consistency level
+        // which is fine to leave null
+        if (keys.isEmpty())
+            return null;
+
+        // Null means no specific consistency behavior is required from Accord, it's functionally similar to
+        // reading at ONE if you are reading data that wasn't written via Accord
+        if (consistencyLevel == null)
+            return null;
+
+        for (Key key : keys)
+        {
+            // readCLForMode should return either null or the supplied consistency level
+            // in which case we will read everything at that CL since Accord doesn't support per table
+            // read consistency
+            ConsistencyLevel readCL = consistencyLevelForAccordRead(cm, key, consistencyLevel);
+            if (readCL != null)
+                return readCL;
+        }
+        return null;
+    }
+
+    private ConsistencyLevel consistencyLevelForAccordRead(ClusterMetadata cm, Key key, ConsistencyLevel consistencyLevel)
+    {
+        // Null means no specific consistency behavior is required from Accord, it's functionally similar to
+        // reading at ONE if you are reading data that wasn't written via Accord
+        if (consistencyLevel == null)
+            return null;
+
+        PartitionKey pk = (PartitionKey)key;
+        TableId tableId = pk.table();
+        Token token = pk.token();
+        TableParams tableParams = getTableMetadata(cm, tableId).params;
+        TransactionalMode mode = tableParams.transactionalMode;
+        TransactionalMigrationFromMode migrationFromMode = tableParams.transactionalMigrationFrom;
+        return mode.readCLForMode(migrationFromMode, consistencyLevel, cm, tableId, token);
+    }
+
+    private static ConsistencyLevel consistencyLevelForAccordCommit(ClusterMetadata cm, Set<Key> keys, @Nullable ConsistencyLevel consistencyLevel)
+    {
+        checkArgument(!keys.isEmpty(), "keys should not be empty");
+        // Null means no specific consistency behavior is required from Accord, it's functionally similar to ANY
+        // if you aren't reading the result back via Accord
+        if (consistencyLevel == null)
+            return null;
+
+        for (Key key : keys)
+        {
+            // commitCLForMode should return either null or the supplied consistency level
+            // in which case we will commit everything at that CL since Accord doesn't support per table
+            // commit consistency
+            ConsistencyLevel commitCL = consistencyLevelForAccordCommit(cm, key, consistencyLevel);
+            if (commitCL != null)
+                return commitCL;
+        }
+        return null;
+    }
+
+    private static ConsistencyLevel consistencyLevelForAccordCommit(ClusterMetadata cm, Key key, @Nullable ConsistencyLevel consistencyLevel)
+    {
+        // Null means no specific consistency behavior is required from Accord, it's functionally similar to ANY
+        // if you aren't reading the result back via Accord
+        if (consistencyLevel == null)
+            return null;
+
+        PartitionKey pk = (PartitionKey)key;
+        TableId tableId = pk.table();
+        Token token = pk.token();
+        TableParams tableParams = getTableMetadata(cm, tableId).params;
+        TransactionalMode mode = tableParams.transactionalMode;
+        TransactionalMigrationFromMode migrationFromMode = tableParams.transactionalMigrationFrom;
+        // commitCLForMode should return either null or the supplied consistency level
+        // in which case we will commit everything at that CL since Accord doesn't support per table
+        // commit consistency
+        return mode.commitCLForMode(migrationFromMode, consistencyLevel, cm, tableId, token);
+    }
+
     @VisibleForTesting
     public Txn createTxn(ClientState state, QueryOptions options)
     {
         SortedSet<Key> keySet = new TreeSet<>();
+        ClusterMetadata cm = ClusterMetadata.current();
 
         if (updates.isEmpty())
         {
@@ -346,18 +443,17 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
             Preconditions.checkState(conditions.isEmpty(), "No condition should exist without updates present");
             List<TxnNamedRead> reads = createNamedReads(options, state, null, keySet::add);
             Keys txnKeys = toKeys(keySet);
-            TxnKeyRead read = createTxnRead(reads, null);
+            TxnRead read = createTxnRead(reads, consistencyLevelForAccordRead(cm, keySet, options.getSerialConsistency()), Domain.Key);
             Txn.Kind kind = shouldReadEphemerally(txnKeys, Schema.instance.getTableMetadata(((AccordRoutableKey) txnKeys.get(0)).table()).params, Read);
             return new Txn.InMemory(kind, txnKeys, read, TxnQuery.ALL, null);
         }
         else
         {
             Int2ObjectHashMap<NamedSelect> autoReads = new Int2ObjectHashMap<>();
-            AccordUpdate update = createUpdate(state, options, autoReads, keySet::add);
+            AccordUpdate update = createUpdate(cm, state, options, autoReads, keySet);
             List<TxnNamedRead> reads = createNamedReads(options, state, autoReads, keySet::add);
-            Keys txnKeys = toKeys(keySet);
-            TxnKeyRead read = createTxnRead(reads, null);
-            return new Txn.InMemory(txnKeys, read, TxnQuery.ALL, update);
+            TxnRead read = createTxnRead(reads, null, Domain.Key);
+            return new Txn.InMemory(toKeys(keySet), read, TxnQuery.ALL, update);
         }
     }
 

src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java

@@ -23,6 +23,7 @@ import java.util.List;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
@@ -30,5 +31,5 @@ import org.apache.cassandra.service.ClientState;
 public interface UpdatesCollector
 {
     PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency);
-    List<IMutation> toMutations(ClientState state, boolean allowPotentialTxnConflicts);
+    List<IMutation> toMutations(ClientState state, PotentialTxnConflicts allowPotentialTxnConflicts);
 }

src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java

@@ -595,7 +595,7 @@ public abstract class AlterTableStatement extends AlterSchemaStatement
                 MemtableParams.get(attrs.getString(TableParams.Option.MEMTABLE.toString()));
             Guardrails.tableProperties.guard(attrs.updatedProperties(), attrs::removeProperty, state);
 
-            validateDefaultTimeToLive(attrs.asNewTableParams());
+            validateDefaultTimeToLive(attrs.asNewTableParams(keyspaceName));
         }
 
         private TableParams validateAndUpdateTransactionalMigration(boolean isCounter, TableParams prev, TableParams next)

src/java/org/apache/cassandra/cql3/statements/schema/CopyTableStatement.java

@@ -236,7 +236,7 @@ public final class CopyTableStatement extends AlterSchemaStatement
                                                  .sum();
             Guardrails.tables.guard(totalUserTables + 1, targetTableName, false, state);
         }
-        validateDefaultTimeToLive(attrs.asNewTableParams());
+        validateDefaultTimeToLive(attrs.asNewTableParams(keyspaceName));
     }
 
     private void maybeCopyIndexes(TableMetadata.Builder builder, TableMetadata sourceTableMeta, KeyspaceMetadata targetKeyspaceMeta)

src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java

@@ -17,13 +17,20 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
-
 import javax.annotation.Nullable;
 
 import com.google.common.collect.ImmutableSet;
-
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.cassandra.audit.AuditLogContext;
@@ -32,26 +39,45 @@ import org.apache.cassandra.auth.DataResource;
 import org.apache.cassandra.auth.IResource;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.cql3.constraints.ColumnConstraints;
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.CQLFragmentParser;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.CqlParser;
+import org.apache.cassandra.cql3.QualifiedName;
 import org.apache.cassandra.cql3.functions.masking.ColumnMask;
 import org.apache.cassandra.db.guardrails.Guardrails;
-import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.db.marshal.EmptyType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UserType;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.service.ClientState;
-import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
+import static com.google.common.collect.Iterables.concat;
 import static java.lang.String.format;
 import static java.util.Comparator.comparing;
 
-import static com.google.common.collect.Iterables.concat;
-
 public final class CreateTableStatement extends AlterSchemaStatement
 {
     private final String tableName;
@@ -189,7 +215,7 @@ public final class CreateTableStatement extends AlterSchemaStatement
         if (useCompactStorage)
             Guardrails.compactTablesEnabled.ensureEnabled(state);
 
-        validateDefaultTimeToLive(attrs.asNewTableParams());
+        validateDefaultTimeToLive(attrs.asNewTableParams(keyspaceName));
 
         rawColumns.forEach((name, raw) -> raw.validate(state, name));
     }
@@ -224,7 +250,7 @@ public final class CreateTableStatement extends AlterSchemaStatement
     public TableMetadata.Builder builder(Types types, UserFunctions functions)
     {
         attrs.validate();
-        TableParams params = attrs.asNewTableParams();
+        TableParams params = attrs.asNewTableParams(keyspaceName);
 
         // use a TreeMap to preserve ordering across JDK versions (see CASSANDRA-9492) - important for stable unit tests
         Map<ColumnIdentifier, ColumnProperties> columns = new TreeMap<>(comparing(o -> o.bytes));

src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java

@@ -17,7 +17,12 @@
  */
 package org.apache.cassandra.cql3.statements.schema;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -27,7 +32,11 @@ import org.apache.cassandra.audit.AuditLogContext;
 import org.apache.cassandra.audit.AuditLogEntryType;
 import org.apache.cassandra.auth.Permission;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.cql3.CQLStatement;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.cql3.VariableSpecifications;
+import org.apache.cassandra.cql3.WhereClause;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.cql3.selection.RawSelector;
 import org.apache.cassandra.cql3.selection.Selectable;
@@ -38,19 +47,24 @@ import org.apache.cassandra.db.marshal.ReversedType;
 import org.apache.cassandra.db.view.View;
 import org.apache.cassandra.exceptions.AlreadyExistsException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.schema.ViewMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Event.SchemaChange;
 import org.apache.cassandra.transport.Event.SchemaChange.Change;
 import org.apache.cassandra.transport.Event.SchemaChange.Target;
 
-import static java.lang.String.join;
-
 import static com.google.common.collect.Iterables.concat;
 import static com.google.common.collect.Iterables.filter;
 import static com.google.common.collect.Iterables.transform;
+import static java.lang.String.join;
 import static org.apache.cassandra.config.CassandraRelevantProperties.MV_ALLOW_FILTERING_NONKEY_COLUMNS_UNSAFE;
 
 public final class CreateViewStatement extends AlterSchemaStatement
@@ -334,7 +348,7 @@ public final class CreateViewStatement extends AlterSchemaStatement
         else if (!builder.hasId())
             builder.id(TableId.get(metadata));
 
-        builder.params(attrs.asNewTableParams())
+        builder.params(attrs.asNewTableParams(keyspaceName))
                .kind(TableMetadata.Kind.VIEW);
 
         partitionKeyColumns.stream()

src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java

@@ -31,6 +31,8 @@ import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.CompressionParams;
 import org.apache.cassandra.schema.MemtableParams;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.schema.TableParams.Option;
@@ -41,7 +43,24 @@ import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 
 import static java.lang.String.format;
-import static org.apache.cassandra.schema.TableParams.Option.*;
+import static org.apache.cassandra.schema.TableParams.Option.ADDITIONAL_WRITE_POLICY;
+import static org.apache.cassandra.schema.TableParams.Option.ALLOW_AUTO_SNAPSHOT;
+import static org.apache.cassandra.schema.TableParams.Option.BLOOM_FILTER_FP_CHANCE;
+import static org.apache.cassandra.schema.TableParams.Option.CACHING;
+import static org.apache.cassandra.schema.TableParams.Option.CDC;
+import static org.apache.cassandra.schema.TableParams.Option.COMMENT;
+import static org.apache.cassandra.schema.TableParams.Option.COMPACTION;
+import static org.apache.cassandra.schema.TableParams.Option.COMPRESSION;
+import static org.apache.cassandra.schema.TableParams.Option.CRC_CHECK_CHANCE;
+import static org.apache.cassandra.schema.TableParams.Option.DEFAULT_TIME_TO_LIVE;
+import static org.apache.cassandra.schema.TableParams.Option.GC_GRACE_SECONDS;
+import static org.apache.cassandra.schema.TableParams.Option.INCREMENTAL_BACKUPS;
+import static org.apache.cassandra.schema.TableParams.Option.MAX_INDEX_INTERVAL;
+import static org.apache.cassandra.schema.TableParams.Option.MEMTABLE_FLUSH_PERIOD_IN_MS;
+import static org.apache.cassandra.schema.TableParams.Option.MIN_INDEX_INTERVAL;
+import static org.apache.cassandra.schema.TableParams.Option.READ_REPAIR;
+import static org.apache.cassandra.schema.TableParams.Option.SPECULATIVE_RETRY;
+import static org.apache.cassandra.schema.TableParams.Option.TRANSACTIONAL_MODE;
 
 public final class TableAttributes extends PropertyDefinitions
 {
@@ -65,10 +84,10 @@ public final class TableAttributes extends PropertyDefinitions
         build(TableParams.builder()).validate();
     }
 
-    TableParams asNewTableParams()
+    TableParams asNewTableParams(String keyspaceName)
     {
         TableParams.Builder builder = TableParams.builder();
-        if (!hasOption(TRANSACTIONAL_MODE))
+        if (!hasOption(TRANSACTIONAL_MODE) && !SchemaConstants.isSystemKeyspace(keyspaceName) && Schema.instance.distributedKeyspaces().names().contains(keyspaceName))
             builder.transactionalMode(DatabaseDescriptor.defaultTransactionalMode());
         return build(builder);
     }

src/java/org/apache/cassandra/db/CounterMutation.java

@@ -36,6 +36,7 @@ import com.google.common.collect.PeekingIterator;
 import com.google.common.util.concurrent.Striped;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.context.CounterContext;
 import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
@@ -189,9 +190,9 @@ public class CounterMutation implements IMutation
      * anyways and it's safe to continue non-transactionally updating them
      */
     @Override
-    public boolean allowsPotentialTransactionConflicts()
+    public PotentialTxnConflicts potentialTxnConflicts()
     {
-        return true;
+        return PotentialTxnConflicts.ALLOW;
     }
 
     private void grabCounterLocks(Keyspace keyspace, List<Lock> locks) throws WriteTimeoutException

src/java/org/apache/cassandra/db/IMutation.java

@@ -24,6 +24,7 @@ import java.util.function.Supplier;
 import javax.annotation.Nullable;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ClientState;
@@ -84,9 +85,9 @@ public interface IMutation
      * like Accord that can't safely read data that is written non-transactionally.
      *
      */
-    default boolean allowsPotentialTransactionConflicts()
+    default PotentialTxnConflicts potentialTxnConflicts()
     {
-        return false;
+        return PotentialTxnConflicts.DISALLOW;
     }
 
     // Construct replacement mutation that is identical except it only includes updates for the specified tables

src/java/org/apache/cassandra/db/MutableDeletionInfo.java

@@ -23,7 +23,8 @@ import java.util.Iterator;
 import com.google.common.base.Objects;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.memory.ByteBufferCloner;
 
@@ -230,7 +231,7 @@ public class MutableDeletionInfo implements DeletionInfo
         return this;
     }
 
-    public DeletionInfo updateAllTimestampAndLocalDeletionTime(long timestamp, int localDeletionTime)
+    public DeletionInfo updateAllTimestampAndLocalDeletionTime(long timestamp, long localDeletionTime)
     {
         if (partitionDeletion.markedForDeleteAt() != Long.MIN_VALUE)
             partitionDeletion = DeletionTime.build(timestamp, localDeletionTime);

src/java/org/apache/cassandra/db/Mutation.java

@@ -39,6 +39,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.DeserializationHelper;
@@ -98,31 +99,31 @@ public class Mutation implements IMutation, Supplier<Mutation>
     // that are only safe to write to from a transaction system.
     // Don't refuse to apply this mutation because it should go through a transaction system
     // because it is being applied by one or in a context where transaction conflicts don't occur
-    private boolean allowPotentialTransactionConflicts;
+    private PotentialTxnConflicts potentialTxnConflicts;
 
     public Mutation(PartitionUpdate update)
     {
-        this(update, false);
+        this(update, PotentialTxnConflicts.DISALLOW);
     }
 
-    public Mutation(PartitionUpdate update, boolean allowPotentialTransactionConflicts)
+    public Mutation(PartitionUpdate update, PotentialTxnConflicts potentialTxnConflicts)
     {
-        this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, allowPotentialTransactionConflicts);
+        this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, potentialTxnConflicts);
     }
 
-    public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean allowPotentialTransactionConflicts)
+    public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, PotentialTxnConflicts potentialTxnConflicts)
     {
-        this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), allowPotentialTransactionConflicts);
+        this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), potentialTxnConflicts);
     }
 
-    public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean cdcEnabled, boolean allowPotentialTransactionConflicts)
+    public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap<TableId, PartitionUpdate> modifications, long approxCreatedAtNanos, boolean cdcEnabled, PotentialTxnConflicts potentialTxnConflicts)
     {
         this.keyspaceName = keyspaceName;
         this.key = key;
         this.modifications = modifications;
         this.cdcEnabled = cdcEnabled;
         this.approxCreatedAtNanos = approxCreatedAtNanos;
-        this.allowPotentialTransactionConflicts = allowPotentialTransactionConflicts;
+        this.potentialTxnConflicts = potentialTxnConflicts;
     }
 
     private static boolean cdcEnabled(Iterable<PartitionUpdate> modifications)
@@ -156,7 +157,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
 
         Map<TableId, PartitionUpdate> updates = builder.build();
         checkState(!updates.isEmpty(), "Updates should not be empty");
-        return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, allowPotentialTransactionConflicts);
+        return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, potentialTxnConflicts);
     }
 
     public @Nullable Mutation without(TableId tableId)
@@ -248,12 +249,12 @@ public class Mutation implements IMutation, Supplier<Mutation>
         Set<TableId> updatedTables = new HashSet<>();
         String ks = null;
         DecoratedKey key = null;
-        Boolean allowPotentialTransactionConflicts = null;
+        PotentialTxnConflicts potentialTxnConflicts = null;
         for (Mutation mutation : mutations)
         {
-            if (allowPotentialTransactionConflicts != null && allowPotentialTransactionConflicts != mutation.allowPotentialTransactionConflicts)
+            if (potentialTxnConflicts != null && potentialTxnConflicts != mutation.potentialTxnConflicts)
                 throw new IllegalArgumentException("Can't merge mutations with differing policies on allowing potential transaction conflicts");
-            allowPotentialTransactionConflicts = mutation.allowPotentialTransactionConflicts;
+            potentialTxnConflicts = mutation.potentialTxnConflicts;
             updatedTables.addAll(mutation.modifications.keySet());
             if (ks != null && !ks.equals(mutation.keyspaceName))
                 throw new IllegalArgumentException();
@@ -280,7 +281,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
             modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates));
             updates.clear();
         }
-        return new Mutation(ks, key, modifications.build(), approxTime.now(), allowPotentialTransactionConflicts);
+        return new Mutation(ks, key, modifications.build(), approxTime.now(), potentialTxnConflicts);
     }
 
     public Future<?> applyFuture()
@@ -339,24 +340,24 @@ public class Mutation implements IMutation, Supplier<Mutation>
 
     public void allowPotentialTransactionConflicts()
     {
-        allowPotentialTransactionConflicts = true;
+        potentialTxnConflicts = PotentialTxnConflicts.ALLOW;
         Arrays.fill(cachedSerializations, null);
     }
 
     @Override
-    public boolean allowsPotentialTransactionConflicts()
+    public PotentialTxnConflicts potentialTxnConflicts()
     {
-        return allowPotentialTransactionConflicts;
+        return potentialTxnConflicts;
     }
 
-    private static int allowPotentialTransactionConflictsFlag(boolean allowPotentialTransactionConflicts)
+    private static int potentialTxnConflictsFlag(PotentialTxnConflicts potentialTxnConflicts)
     {
-        return allowPotentialTransactionConflicts ? ALLOW_POTENTIAL_TRANSACTION_CONFLICTS : 0;
+        return potentialTxnConflicts.allowed ? ALLOW_POTENTIAL_TRANSACTION_CONFLICTS : 0;
     }
 
-    public static boolean allowPotentialTransactionConflicts(int flags)
+    public static PotentialTxnConflicts potentialTxnConflicts(int flags)
     {
-        return (flags & ALLOW_POTENTIAL_TRANSACTION_CONFLICTS) != 0;
+        return (flags & ALLOW_POTENTIAL_TRANSACTION_CONFLICTS) != 0 ? PotentialTxnConflicts.ALLOW : PotentialTxnConflicts.DISALLOW;
     }
 
     public String toString()
@@ -443,7 +444,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
          * being handled by the caller
          * @return this builder
          */
-        public SimpleBuilder allowPotentialTransactionConflicts();
+        public SimpleBuilder allowPotentialTxnConflicts();
 
         /**
          * Sets the timestamp to use for the following additions to this builder or any derived (update or row) builder.
@@ -560,7 +561,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
             if (version >= VERSION_51)
             {
                 int flags = 0;
-                flags |= allowPotentialTransactionConflictsFlag(mutation.allowPotentialTransactionConflicts);
+                flags |= potentialTxnConflictsFlag(mutation.potentialTxnConflicts);
                 out.write(flags);
             }
 
@@ -583,11 +584,11 @@ public class Mutation implements IMutation, Supplier<Mutation>
             {
                 teeIn = new TeeDataInputPlus(in, dob, CACHEABLE_MUTATION_SIZE_LIMIT);
 
-                boolean allowPotentialTransactionConflicts = false;
+                PotentialTxnConflicts potentialTxnConflicts = PotentialTxnConflicts.DISALLOW;
                 if (version >= VERSION_51)
                 {
                     int flags = teeIn.readByte();
-                    allowPotentialTransactionConflicts = allowPotentialTransactionConflicts(flags);
+                    potentialTxnConflicts = potentialTxnConflicts(flags);
                 }
                 int size = teeIn.readUnsignedVInt32();
                 assert size > 0;
@@ -595,7 +596,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
                 PartitionUpdate update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
                 if (size == 1)
                 {
-                    m = new Mutation(update, allowPotentialTransactionConflicts);
+                    m = new Mutation(update, potentialTxnConflicts);
                 }
                 else
                 {
@@ -608,7 +609,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
                         update = PartitionUpdate.serializer.deserialize(teeIn, version, flag);
                         modifications.put(update.metadata().id, update);
                     }
-                    m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), allowPotentialTransactionConflicts);
+                    m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), potentialTxnConflicts);
                 }
 
                 //Only cache serializations that don't hit the limit
@@ -708,18 +709,18 @@ public class Mutation implements IMutation, Supplier<Mutation>
         private final long approxCreatedAtNanos = approxTime.now();
         private boolean empty = true;
 
-        private boolean allowPotentialTransactionConflicts;
+        private PotentialTxnConflicts potentialTxnConflicts;
 
         public PartitionUpdateCollector(String keyspaceName, DecoratedKey key)
         {
-            this(keyspaceName, key, false);
+            this(keyspaceName, key, PotentialTxnConflicts.DISALLOW);
         }
 
-        public PartitionUpdateCollector(String keyspaceName, DecoratedKey key, boolean allowPotentialTransactionConflicts)
+        public PartitionUpdateCollector(String keyspaceName, DecoratedKey key, PotentialTxnConflicts potentialTxnConflicts)
         {
             this.keyspaceName = keyspaceName;
             this.key = key;
-            this.allowPotentialTransactionConflicts = allowPotentialTransactionConflicts;
+            this.potentialTxnConflicts = potentialTxnConflicts;
         }
 
         public PartitionUpdateCollector add(PartitionUpdate partitionUpdate)
@@ -753,7 +754,7 @@ public class Mutation implements IMutation, Supplier<Mutation>
 
         public Mutation build()
         {
-            return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, allowPotentialTransactionConflicts);
+            return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, potentialTxnConflicts);
         }
     }
 }

src/java/org/apache/cassandra/db/MutationVerbHandler.java

@@ -19,7 +19,10 @@ package org.apache.cassandra.db;
 
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.*;
+import org.apache.cassandra.net.ForwardingInfo;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.ParamType;
 import org.apache.cassandra.tracing.Tracing;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;

src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java

@@ -72,7 +72,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                         boolean isDigest,
                                         int digestVersion,
                                         boolean acceptsTransient,
-                                        boolean allowOutOfRangeReads,
+                                        PotentialTxnConflicts potentialTxnConflicts,
                                         TableMetadata metadata,
                                         long nowInSec,
                                         ColumnFilter columnFilter,
@@ -82,7 +82,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                         Index.QueryPlan indexQueryPlan,
                                         boolean trackWarnings)
     {
-        super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, allowOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
+        super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
         this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata());
     }
 
@@ -90,7 +90,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                     boolean isDigest,
                                                     int digestVersion,
                                                     boolean acceptsTransient,
-                                                    boolean allowsOutOfRangeReads,
+                                                    PotentialTxnConflicts potentialTxnConflicts,
                                                     TableMetadata metadata,
                                                     long nowInSec,
                                                     ColumnFilter columnFilter,
@@ -118,7 +118,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                              isDigest,
                                              digestVersion,
                                              acceptsTransient,
-                                             allowsOutOfRangeReads,
+                                             potentialTxnConflicts,
                                              metadata,
                                              nowInSec,
                                              columnFilter,
@@ -140,7 +140,30 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       false,
                       0,
                       false,
+                      PotentialTxnConflicts.DISALLOW,
+                      metadata,
+                      nowInSec,
+                      columnFilter,
+                      rowFilter,
+                      limits,
+                      dataRange,
+                      findIndexQueryPlan(metadata, rowFilter),
+                      false);
+    }
+
+    public static PartitionRangeReadCommand create(TableMetadata metadata,
+                                                   long nowInSec,
+                                                   ColumnFilter columnFilter,
+                                                   RowFilter rowFilter,
+                                                   DataLimits limits,
+                                                   DataRange dataRange,
+                                                   PotentialTxnConflicts potentialTxnConflicts)
+    {
+        return create(metadata.epoch,
                       false,
+                      0,
+                      false,
+                      potentialTxnConflicts,
                       metadata,
                       nowInSec,
                       columnFilter,
@@ -165,7 +188,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       false,
                       0,
                       false,
-                      false,
+                      PotentialTxnConflicts.DISALLOW,
                       metadata,
                       nowInSec,
                       ColumnFilter.all(metadata),
@@ -212,7 +235,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -223,32 +246,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isTrackingWarnings());
     }
 
-    /*
-     * The execution method does not need to perform reconciliation so the read command
-     * should execute in a mannager suited to not needing reconciliation. Such as when
-     * executing transactionally at a single replica and doing an index scan where the index
-     * scan should not return extra rows and expect post filtering at the coordinator.
-     */
-    public PartitionRangeReadCommand withoutReconciliation()
-    {
-        if (rowFilter().isEmpty())
-            return this;
-        return create(serializedAtEpoch(),
-                      isDigestQuery(),
-                      digestVersion(),
-                      acceptsTransient(),
-                      allowsOutOfRangeReads(),
-                      metadata(),
-                      nowInSec(),
-                      columnFilter(),
-                      rowFilter().withoutReconciliation(),
-                      limits(),
-                      dataRange(),
-                      indexQueryPlan(),
-                      isTrackingWarnings());
-    }
-
-    public PartitionRangeReadCommand forSubRangeWithNowInSeconds(long nowInSec, AbstractBounds<PartitionPosition> range, boolean isRangeContinuation)
+    public PartitionRangeReadCommand withTransactionalSettings(long nowInSec, AbstractBounds<PartitionPosition> range, boolean isRangeContinuation, boolean withoutReconciliation)
     {
         // If we're not a continuation of whatever range we've previously queried, we should ignore the states of the
         // DataLimits as it's either useless, or misleading. This is particularly important for GROUP BY queries, where
@@ -259,26 +257,26 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      PotentialTxnConflicts.ALLOW,
                       metadata(),
                       nowInSec,
                       columnFilter(),
-                      rowFilter(),
+                      withoutReconciliation ? rowFilter().withoutReconciliation() : rowFilter(),
                       isRangeContinuation ? limits() : limits().withoutState(),
                       dataRange().forSubRange(range),
                       indexQueryPlan(),
                       isTrackingWarnings());
     }
 
-    public PartitionRangeReadCommand withNowInSec(long nowInSec)
+    public PartitionRangeReadCommand withTxnReadName(int txnReadName)
     {
         return create(serializedAtEpoch(),
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
-                      nowInSec,
+                      txnReadName,
                       columnFilter(),
                       rowFilter(),
                       limits(),
@@ -293,7 +291,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -311,7 +309,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       true,
                       digestVersion(),
                       false,
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -329,7 +327,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       false,
                       0,
                       true,
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -347,7 +345,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -365,7 +363,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -601,7 +599,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                        boolean isDigest,
                                        int digestVersion,
                                        boolean acceptsTransient,
-                                       boolean allowsOutOfRangeReads,
+                                       PotentialTxnConflicts potentialTxnConflicts,
                                        TableMetadata metadata,
                                        long nowInSec,
                                        ColumnFilter columnFilter,
@@ -611,7 +609,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
         throws IOException
         {
             DataRange range = DataRange.serializer.deserialize(in, version, metadata);
-            return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
+            return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
         }
     }
 
@@ -629,7 +627,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                       Index.QueryPlan indexQueryPlan,
                                                       boolean trackWarnings)
         {
-            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, true, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
+            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
         }
 
         @Override

src/java/org/apache/cassandra/db/PartitionRangeReadQuery.java

@@ -17,6 +17,7 @@
  */
 package org.apache.cassandra.db;
 
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
@@ -36,9 +37,10 @@ public interface PartitionRangeReadQuery extends ReadQuery
                             ColumnFilter columnFilter,
                             RowFilter rowFilter,
                             DataLimits limits,
-                            DataRange dataRange)
+                            DataRange dataRange,
+                            PotentialTxnConflicts potentialTxnConflicts)
     {
-        return PartitionRangeReadCommand.create(table, nowInSec, columnFilter, rowFilter, limits, dataRange);
+        return PartitionRangeReadCommand.create(table, nowInSec, columnFilter, rowFilter, limits, dataRange, potentialTxnConflicts);
     }
 
     /**

src/java/org/apache/cassandra/db/RangeTombstoneList.java

@@ -22,14 +22,14 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.utils.AbstractIterator;
-import org.apache.cassandra.utils.CassandraUInt;
-
 import com.google.common.collect.Iterators;
 
 import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.utils.AbstractIterator;
+import org.apache.cassandra.utils.CassandraUInt;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.memory.ByteBufferCloner;
 
@@ -328,12 +328,13 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
             markedAts[i] = timestamp;
     }
 
-    public void updateAllTimestampAndLocalDeletionTime(long timestamp, int localDeletionTime)
+    public void updateAllTimestampAndLocalDeletionTime(long timestamp, long localDeletionTime)
     {
+        int unsignedLocalDeletionTime = Cell.deletionTimeLongToUnsignedInteger(localDeletionTime);
         for (int i = 0; i < size; i++)
         {
             markedAts[i] = timestamp;
-            delTimesUnsignedIntegers[i] = localDeletionTime;
+            delTimesUnsignedIntegers[i] = unsignedLocalDeletionTime;
         }
     }
 

src/java/org/apache/cassandra/db/ReadCommand.java

@@ -90,6 +90,7 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.service.ClientWarn;
+import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
@@ -100,6 +101,7 @@ import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.TimeUUID;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
 import static org.apache.cassandra.db.partitions.UnfilteredPartitionIterators.MergeListener.NOOP;
@@ -119,6 +121,30 @@ public abstract class ReadCommand extends AbstractReadQuery
     protected static final Logger logger = LoggerFactory.getLogger(ReadCommand.class);
     public static final IVersionedSerializer<ReadCommand> serializer = new Serializer();
 
+    public enum PotentialTxnConflicts
+    {
+        /**
+         * Check for and raise an error if this operation should have been transactionally managed. For use
+         * by queries that aren't issued by a transaction system managing potential conflicts in contexts where
+         * conflicts would be a problem.
+         */
+        DISALLOW(false),
+
+        /**
+         * Don't check or raise an error if this operation could conflict with transactions. For use when the thing
+         * being managed doesn't support transactions or the operation is being done by a transaction that is already
+         * managing any potential conflicts.
+         */
+        ALLOW(true);
+
+        public final boolean allowed;
+
+        PotentialTxnConflicts(boolean allowed)
+        {
+            this.allowed = allowed;
+        }
+    }
+    
     // Expose the active command running so transitive calls can lookup this command.
     // This is useful for a few reasons, but mainly because the CQL query is here.
     private static final FastThreadLocal<ReadCommand> COMMAND = new FastThreadLocal<>();
@@ -128,7 +154,8 @@ public abstract class ReadCommand extends AbstractReadQuery
     private final boolean isDigestQuery;
     private final boolean acceptsTransient;
     private final Epoch serializedAtEpoch;
-    private boolean allowsOutOfRangeReads;
+    private final PotentialTxnConflicts potentialTxnConflicts;
+
     // if a digest query, the version for which the digest is expected. Ignored if not a digest.
     private int digestVersion;
 
@@ -147,7 +174,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                                                 boolean isDigest,
                                                 int digestVersion,
                                                 boolean acceptsTransient,
-                                                boolean allowsOutOfRangeReads,
+                                                PotentialTxnConflicts potentialTxnConflicts,
                                                 TableMetadata metadata,
                                                 long nowInSec,
                                                 ColumnFilter columnFilter,
@@ -174,7 +201,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                           boolean isDigestQuery,
                           int digestVersion,
                           boolean acceptsTransient,
-                          boolean allowsOutOfRangeReads,
+                          PotentialTxnConflicts potentialTxnConflicts,
                           TableMetadata metadata,
                           long nowInSec,
                           ColumnFilter columnFilter,
@@ -193,7 +220,7 @@ public abstract class ReadCommand extends AbstractReadQuery
         this.digestVersion = digestVersion;
         this.acceptsTransient = acceptsTransient;
         this.indexQueryPlan = indexQueryPlan;
-        this.allowsOutOfRangeReads = allowsOutOfRangeReads;
+        this.potentialTxnConflicts = potentialTxnConflicts;
         this.trackWarnings = trackWarnings;
         this.serializedAtEpoch = serializedAtEpoch;
         this.dataRange = dataRange;
@@ -345,7 +372,7 @@ public abstract class ReadCommand extends AbstractReadQuery
      */
     public ReadCommand copyAsTransientQuery(Replica replica)
     {
-        Preconditions.checkArgument(replica.isTransient(),
+        checkArgument(replica.isTransient(),
                                     "Can't make a transient request on a full replica: " + replica);
         return copyAsTransientQuery();
     }
@@ -367,7 +394,7 @@ public abstract class ReadCommand extends AbstractReadQuery
      */
     public ReadCommand copyAsDigestQuery(Replica replica)
     {
-        Preconditions.checkArgument(replica.isFull(),
+        checkArgument(replica.isFull(),
                                     "Can't make a digest request on a transient replica " + replica);
         return copyAsDigestQuery();
     }
@@ -462,6 +489,8 @@ public abstract class ReadCommand extends AbstractReadQuery
         try
         {
             ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata());
+            if (!potentialTxnConflicts.allowed)
+                ConsensusRequestRouter.validateSafeToReadNonTransactionally(this);
             Index.QueryPlan indexQueryPlan = indexQueryPlan();
 
             Index.Searcher searcher = null;
@@ -550,15 +579,9 @@ public abstract class ReadCommand extends AbstractReadQuery
         return ReadExecutionController.forCommand(this, false);
     }
 
-    public ReadCommand allowOutOfRangeReads()
+    public PotentialTxnConflicts potentialTxnConflicts()
     {
-        allowsOutOfRangeReads = true;
-        return this;
-    }
-
-    public boolean allowsOutOfRangeReads()
-    {
-        return allowsOutOfRangeReads;
+        return potentialTxnConflicts;
     }
 
     /**
@@ -1250,7 +1273,7 @@ public abstract class ReadCommand extends AbstractReadQuery
         private static final int HAS_INDEX = 0x04;
         private static final int ACCEPTS_TRANSIENT = 0x08;
         private static final int NEEDS_RECONCILIATION = 0x10;
-        private static final int ALLOWS_OUT_OF_RANGE_READS = 0x20;
+        private static final int ALLOWS_POTENTIAL_TXN_CONFLICTS = 0x20;
 
         private final SchemaProvider schema;
 
@@ -1315,14 +1338,14 @@ public abstract class ReadCommand extends AbstractReadQuery
             return (flags & NEEDS_RECONCILIATION) != 0;
         }
 
-        private static int allowsOutOfRangeReadsFlag(boolean allowsOutOfRangeReads)
+        private static int potentialTxnConflicts(PotentialTxnConflicts potentialTxnConflicts)
         {
-            return allowsOutOfRangeReads ? ALLOWS_OUT_OF_RANGE_READS: 0;
+            return potentialTxnConflicts.allowed ? ALLOWS_POTENTIAL_TXN_CONFLICTS : 0;
         }
 
-        private static boolean allowsOutOfRangeReads(int flags)
+        private static PotentialTxnConflicts potentialTxnConflicts(int flags)
         {
-            return (flags & ALLOWS_OUT_OF_RANGE_READS) != 0;
+            return (flags & ALLOWS_POTENTIAL_TXN_CONFLICTS) != 0 ? PotentialTxnConflicts.ALLOW : PotentialTxnConflicts.DISALLOW;
         }
 
         public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
@@ -1333,7 +1356,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                     | indexFlag(null != command.indexQueryPlan())
                     | acceptsTransientFlag(command.acceptsTransient())
                     | needsReconciliationFlag(command.rowFilter().needsReconciliation())
-                    | allowsOutOfRangeReadsFlag(command.allowsOutOfRangeReads)
+                    | potentialTxnConflicts(command.potentialTxnConflicts)
             );
             if (command.isDigestQuery())
                 out.writeUnsignedVInt32(command.digestVersion());
@@ -1359,7 +1382,7 @@ public abstract class ReadCommand extends AbstractReadQuery
             int flags = in.readByte();
             boolean isDigest = isDigest(flags);
             boolean acceptsTransient = acceptsTransient(flags);
-            boolean allowsOutOfRangeReads = allowsOutOfRangeReads(flags);
+            PotentialTxnConflicts potentialTxnConflicts = potentialTxnConflicts(flags);
             // Shouldn't happen or it's a user error (see comment above) but
             // better complain loudly than doing the wrong thing.
             if (isForThrift(flags))
@@ -1405,7 +1428,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                     indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
             }
 
-            return kind.selectionDeserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
+            return kind.selectionDeserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
         }
 
         private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException

src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java

@@ -28,6 +28,7 @@ import org.apache.cassandra.exceptions.CoordinatorBehindException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.InvalidRoutingException;
 import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.metrics.TCMMetrics;
 import org.apache.cassandra.net.IVerbHandler;
@@ -42,6 +43,7 @@ import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
 
 public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
 {
@@ -99,6 +101,13 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
             MessagingService.instance().send(reply, message.from());
             return;
         }
+        catch (RetryOnDifferentSystemException e)
+        {
+            logger.debug("Responding with retry on different system");
+            MessagingService.instance().respondWithFailure(RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM, message);
+            Tracing.trace("Payload application resulted in RetryOnDifferentSysten");
+            return;
+        }
         catch (AssertionError t)
         {
             throw new AssertionError(String.format("Caught an error while trying to process the command: %s", command.toCQLString()), t);
@@ -164,7 +173,7 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
 
         // Some read commands may be sent using an older Epoch intentionally so validating using the current Epoch
         // doesn't work
-        if (command.allowsOutOfRangeReads())
+        if (command.potentialTxnConflicts().allowed)
             return metadata;
 
         if (command.isTopK())

src/java/org/apache/cassandra/db/ReadResponse.java

@@ -17,14 +17,19 @@
  */
 package org.apache.cassandra.db;
 
-import java.io.*;
+import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -132,6 +137,37 @@ public abstract class ReadResponse
         return sb.toString();
     }
 
+    /**
+     * For range reads Accord generates multiple responses per node because each command store executes
+     * the reads independently. The responses are already sorted in token order so the iterators or digests can be
+     * merged and still produce a consistent result across different nodes.
+     *
+     * This can *only* be called from the node producing the results not the coordinator because isEmptyDigest is
+     * not serialized
+     */
+    public static ReadResponse merge(List<ReadResponse> responses, ReadCommand command)
+    {
+        if (responses.get(0).isDigestResponse())
+        {
+            Digest digest = Digest.forReadResponse();
+            for (ReadResponse response : responses)
+                digest.update(((DigestResponse)response).digest);
+            return new DigestResponse(ByteBuffer.wrap(digest.digest()));
+        }
+        else
+        {
+            List<UnfilteredPartitionIterator> iterators = new ArrayList<>(responses.size());
+            for (ReadResponse response : responses)
+                iterators.add(response.makeIterator(command));
+
+            // Range responses will not respect the limit because each command store returns a separate response
+            // so we effectively deserialize and then reserialize in order to apply the limits
+            // Wasteful, but better than sending it to the coordinator to do it
+            UnfilteredPartitionIterator filtered = command.limits().filter(UnfilteredPartitionIterators.concat(iterators), 0, command.selectsFullPartition());
+            return new LocalDataResponse(filtered, command, NO_OP_REPAIRED_DATA_INFO);
+        }
+    }
+
     protected static ByteBuffer makeDigest(UnfilteredPartitionIterator iterator, ReadCommand command)
     {
         Digest digest = Digest.forReadResponse();

src/java/org/apache/cassandra/db/SimpleBuilders.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.context.CounterContext;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
@@ -120,7 +121,7 @@ public abstract class SimpleBuilders
 
         private final Map<TableId, PartitionUpdateBuilder> updateBuilders = new HashMap<>();
 
-        private boolean allowPotentialTransactionConflicts = false;
+        private PotentialTxnConflicts potentialTxnConflicts = PotentialTxnConflicts.DISALLOW;
 
         public MutationBuilder(String keyspaceName, DecoratedKey key)
         {
@@ -128,9 +129,9 @@ public abstract class SimpleBuilders
             this.key = key;
         }
 
-        public MutationBuilder allowPotentialTransactionConflicts()
+        public MutationBuilder allowPotentialTxnConflicts()
         {
-            allowPotentialTransactionConflicts = true;
+            potentialTxnConflicts = PotentialTxnConflicts.ALLOW;
             return this;
         }
 
@@ -162,9 +163,9 @@ public abstract class SimpleBuilders
             assert !updateBuilders.isEmpty() : "Cannot create empty mutation";
 
             if (updateBuilders.size() == 1)
-                return new Mutation(updateBuilders.values().iterator().next().build(), allowPotentialTransactionConflicts);
+                return new Mutation(updateBuilders.values().iterator().next().build(), potentialTxnConflicts);
 
-            Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key, allowPotentialTransactionConflicts);
+            Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key, potentialTxnConflicts);
             for (PartitionUpdateBuilder builder : updateBuilders.values())
                 mutationBuilder.add(builder.build());
             return mutationBuilder.build();

src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java

@@ -98,7 +98,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                          boolean isDigest,
                                          int digestVersion,
                                          boolean acceptsTransient,
-                                         boolean allowsOutOfRangeReads,
+                                         PotentialTxnConflicts potentialTxnConflicts,
                                          TableMetadata metadata,
                                          long nowInSec,
                                          ColumnFilter columnFilter,
@@ -110,7 +110,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                          boolean trackWarnings,
                                          DataRange dataRange)
     {
-        super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
+        super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
         assert partitionKey.getPartitioner() == metadata.partitioner;
         this.partitionKey = partitionKey;
         this.clusteringIndexFilter = clusteringIndexFilter;
@@ -120,7 +120,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                      boolean isDigest,
                                                      int digestVersion,
                                                      boolean acceptsTransient,
-                                                     boolean allowsOutOfRangeReads,
+                                                     PotentialTxnConflicts potentialTxnConflicts,
                                                      TableMetadata metadata,
                                                      long nowInSec,
                                                      ColumnFilter columnFilter,
@@ -154,7 +154,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                               isDigest,
                                               digestVersion,
                                               acceptsTransient,
-                                              allowsOutOfRangeReads,
+                                              potentialTxnConflicts,
                                               metadata,
                                               nowInSec,
                                               columnFilter,
@@ -194,7 +194,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       false,
                       0,
                       false,
-                      false,
+                      PotentialTxnConflicts.DISALLOW,
                       metadata,
                       nowInSec,
                       columnFilter,
@@ -206,6 +206,44 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       false);
     }
 
+    /**
+     * Creates a new read command on a single partition.
+     *
+     * @param metadata the table to query.
+     * @param nowInSec the time in seconds to use are "now" for this query.
+     * @param columnFilter the column filter to use for the query.
+     * @param rowFilter the row filter to use for the query.
+     * @param limits the limits to use for the query.
+     * @param partitionKey the partition key for the partition to query.
+     * @param clusteringIndexFilter the clustering index filter to use for the query.
+     *
+     * @return a newly created read command.
+     */
+    public static SinglePartitionReadCommand create(TableMetadata metadata,
+                                                    long nowInSec,
+                                                    ColumnFilter columnFilter,
+                                                    RowFilter rowFilter,
+                                                    DataLimits limits,
+                                                    DecoratedKey partitionKey,
+                                                    ClusteringIndexFilter clusteringIndexFilter,
+                                                    PotentialTxnConflicts potentialTxnConflicts)
+    {
+        return create(metadata.epoch,
+                      false,
+                      0,
+                      false,
+                      potentialTxnConflicts,
+                      metadata,
+                      nowInSec,
+                      columnFilter,
+                      rowFilter,
+                      limits,
+                      partitionKey,
+                      clusteringIndexFilter,
+                      findIndexQueryPlan(metadata, rowFilter),
+                      false);
+    }
+
     /**
      * Creates a new read command on a single partition.
      *
@@ -373,7 +411,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -392,7 +430,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       true,
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -411,7 +449,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       false,
                       0,
                       true,
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -430,7 +468,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      potentialTxnConflicts(),
                       metadata(),
                       nowInSec(),
                       columnFilter(),
@@ -442,24 +480,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       isTrackingWarnings());
     }
 
-    public SinglePartitionReadCommand withNowInSec(long nowInSec)
-    {
-        return create(serializedAtEpoch(),
-                      isDigestQuery(),
-                      digestVersion(),
-                      acceptsTransient(),
-                      allowsOutOfRangeReads(),
-                      metadata(),
-                      nowInSec,
-                      columnFilter(),
-                      rowFilter(),
-                      limits(),
-                      partitionKey(),
-                      clusteringIndexFilter(),
-                      indexQueryPlan(),
-                      isTrackingWarnings());
-    }
-
     @Override
     public DecoratedKey partitionKey()
     {
@@ -1274,24 +1294,21 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     }
 
     /*
-     * The execution method does not need to perform reconciliation so the read command
-     * should execute in a mannager suited to not needing reconciliation. Such as when
-     * executing transactionally at a single replica and doing an index scan where the index
-     * scan should not return extra rows and expect post filtering at the coordinator.
+     * When running transactionally we need to use the txn system nowInSeconds, and set whether reconciliation
+     * should be performed based on whether it's part of a multiple replica read. We also allow potential txn conflicts
+     * because we manage those conflicts from the txn system
      */
-    public SinglePartitionReadCommand withoutReconciliation()
+    public SinglePartitionReadCommand withTransactionalSettings(boolean withoutReconciliation, long nowInSeconds)
     {
-        if (rowFilter().isEmpty())
-            return this;
         return create(serializedAtEpoch(),
                       isDigestQuery(),
                       digestVersion(),
                       acceptsTransient(),
-                      allowsOutOfRangeReads(),
+                      PotentialTxnConflicts.ALLOW,
                       metadata(),
-                      nowInSec(),
+                      nowInSeconds,
                       columnFilter(),
-                      rowFilter().withoutReconciliation(),
+                      withoutReconciliation ? rowFilter().withoutReconciliation() : rowFilter(),
                       limits(),
                       partitionKey(),
                       clusteringIndexFilter(),
@@ -1310,7 +1327,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                    RowFilter rowFilter,
                                    DataLimits limits,
                                    List<DecoratedKey> partitionKeys,
-                                   ClusteringIndexFilter clusteringIndexFilter)
+                                   ClusteringIndexFilter clusteringIndexFilter,
+                                   PotentialTxnConflicts potentialTxnConflicts)
         {
             List<SinglePartitionReadCommand> commands = new ArrayList<>(partitionKeys.size());
             for (DecoratedKey partitionKey : partitionKeys)
@@ -1321,7 +1339,8 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                                rowFilter,
                                                                limits,
                                                                partitionKey,
-                                                               clusteringIndexFilter));
+                                                               clusteringIndexFilter,
+                                                               potentialTxnConflicts));
             }
 
             return create(commands, limits);
@@ -1377,7 +1396,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                        boolean isDigest,
                                        int digestVersion,
                                        boolean acceptsTransient,
-                                       boolean allowsOutOfRangeReads,
+                                       PotentialTxnConflicts potentialTxnConflicts,
                                        TableMetadata metadata,
                                        long nowInSec,
                                        ColumnFilter columnFilter,
@@ -1388,7 +1407,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
         {
             DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readBuffer(in, DatabaseDescriptor.getMaxValueSize()));
             ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
-            return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, allowsOutOfRangeReads, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
+            return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
         }
     }
 
@@ -1436,7 +1455,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                          boolean trackWarnings,
                                                          DataRange dataRange)
         {
-            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, true, metadata, nowInSec, columnFilter, 
+            super(metadata.epoch, isDigest, digestVersion, acceptsTransient, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter, 
                   rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings, dataRange);
         }
 

src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java

@@ -23,9 +23,9 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.Iterables;
-
 import org.apache.commons.lang3.tuple.Pair;
 
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.filter.ClusteringIndexFilter;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
@@ -51,9 +51,10 @@ public interface SinglePartitionReadQuery extends ReadQuery
                                                                         RowFilter rowFilter,
                                                                         DataLimits limits,
                                                                         List<DecoratedKey> partitionKeys,
-                                                                        ClusteringIndexFilter clusteringIndexFilter)
+                                                                        ClusteringIndexFilter clusteringIndexFilter,
+                                                                        PotentialTxnConflicts potentialTxnConflicts)
     {
-        return SinglePartitionReadCommand.Group.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKeys, clusteringIndexFilter);
+        return SinglePartitionReadCommand.Group.create(metadata, nowInSec, columnFilter, rowFilter, limits, partitionKeys, clusteringIndexFilter, potentialTxnConflicts);
     }
 
 

src/java/org/apache/cassandra/db/partitions/PartitionIterators.java

@@ -17,16 +17,16 @@
  */
 package org.apache.cassandra.db.partitions;
 
-import java.util.*;
+import java.util.List;
 
 import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.SinglePartitionReadQuery;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.RowIterators;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.utils.AbstractIterator;
 
-import org.apache.cassandra.db.SinglePartitionReadQuery;
-import org.apache.cassandra.db.rows.*;
-
 public abstract class PartitionIterators
 {
     private PartitionIterators() {}
@@ -57,7 +57,7 @@ public abstract class PartitionIterators
         return Transformation.apply(toReturn, new Close());
     }
 
-    public static PartitionIterator concat(final List<PartitionIterator> iterators)
+    public static PartitionIterator concat(final List<? extends PartitionIterator> iterators)
     {
         if (iterators.size() == 1)
             return iterators.get(0);

src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java

@@ -71,14 +71,14 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.metrics.TCMMetrics;
-import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.utils.btree.BTree;
 import org.apache.cassandra.utils.btree.UpdateFunction;
 import org.apache.cassandra.utils.vint.VIntCoding;
@@ -1161,7 +1161,7 @@ public class PartitionUpdate extends AbstractBTreePartition
             return this;
         }
 
-        public Builder updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime)
+        public Builder updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime)
         {
             deletionInfo.updateAllTimestampAndLocalDeletionTime(newTimestamp - 1, newLocalDeletionTime);
             tree = BTree.<Row, Row>transformAndFilter(tree, (x) -> x.updateTimesAndPathsForAccord(cellToMaybeNewListPath, newTimestamp, newLocalDeletionTime));

src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java

@@ -19,11 +19,21 @@ package org.apache.cassandra.db.partitions;
 
 import java.io.IOError;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.NoSuchElementException;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Digest;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.transform.FilteredPartitions;
 import org.apache.cassandra.db.transform.MorePartitions;
 import org.apache.cassandra.db.transform.Transformation;

src/java/org/apache/cassandra/db/rows/AbstractCell.java

@@ -121,14 +121,14 @@ public abstract class AbstractCell<V> extends Cell<V>
     }
 
     @Override
-    public ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime)
+    public ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime)
     {
         long localDeletionTime = localDeletionTime() != NO_DELETION_TIME ? newLocalDeletionTime : NO_DELETION_TIME;
         return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime, buffer(), path());
     }
 
     @Override
-    public Cell<?> updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime)
+    public Cell<?> updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, long newLocalDeletionTime)
     {
         long localDeletionTime = localDeletionTime() != NO_DELETION_TIME ? newLocalDeletionTime : NO_DELETION_TIME;
         return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime, buffer(), maybeNewPath);

src/java/org/apache/cassandra/db/rows/BTreeRow.java

@@ -444,7 +444,7 @@ public class BTreeRow extends AbstractRow
     }
 
     @Override
-    public Row updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime)
+    public Row updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime)
     {
         LivenessInfo newInfo = primaryKeyLivenessInfo.isEmpty() ? primaryKeyLivenessInfo : primaryKeyLivenessInfo.withUpdatedTimestampAndLocalDeletionTime(newTimestamp, newLocalDeletionTime);
         // If the deletion is shadowable and the row has a timestamp, we'll forced the deletion timestamp to be less than the row one, so we

src/java/org/apache/cassandra/db/rows/ColumnData.java

@@ -291,7 +291,7 @@ public abstract class ColumnData implements IMeasurableMemory
     /**
      * @param cellToMaybeNewListPath If the cell is a list append cell a new cell path is returned generated based on the Accord executeAt timestamp
      */
-    public abstract ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime);
+    public abstract ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime);
 
     /**
      * List paths are time UUIDs that increment for each item in the list and for Accord and Paxos
@@ -299,7 +299,7 @@ public abstract class ColumnData implements IMeasurableMemory
      *
      * @param maybeNewPath If this cell is a list append for a non-frozen list (multi-cell) then it will be new path generated using the executeAt timestamp, otherwise it will be the existing path
      */
-    public abstract ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime);
+    public abstract ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, long newLocalDeletionTime);
 
     public abstract ColumnData markCounterLocalToBeCleared();
 

src/java/org/apache/cassandra/db/rows/ComplexColumnData.java

@@ -267,7 +267,7 @@ public class ComplexColumnData extends ColumnData implements Iterable<Cell<?>>
     }
 
     @Override
-    public ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime)
+    public ColumnData updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime)
     {
         DeletionTime newDeletion = complexDeletion.isLive() ? complexDeletion : DeletionTime.build(newTimestamp - 1, newLocalDeletionTime);
         Function<Cell, CellPath> maybeNewListPath;
@@ -279,7 +279,7 @@ public class ComplexColumnData extends ColumnData implements Iterable<Cell<?>>
     }
 
     @Override
-    public ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, int newLocalDeletionTime)
+    public ColumnData updateAllTimesWithNewCellPathForComplexColumnData(@Nonnull CellPath maybeNewPath, long newTimestamp, long newLocalDeletionTime)
     {
         throw new UnsupportedOperationException();
     }

src/java/org/apache/cassandra/db/rows/Row.java

@@ -312,7 +312,7 @@ public interface Row extends Unfiltered, Iterable<ColumnData>, IMeasurableMemory
      */
     public Row updateAllTimestamp(long newTimestamp);
 
-    public Row updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, int newLocalDeletionTime);
+    public Row updateTimesAndPathsForAccord(@Nonnull Function<Cell, CellPath> cellToMaybeNewListPath, long newTimestamp, long newLocalDeletionTime);
 
     /**
      * Returns a copy of this row with the new deletion as row deletion if it is more recent

src/java/org/apache/cassandra/db/virtual/VirtualMutation.java

@@ -30,6 +30,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IMutation;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.ClientState;
@@ -142,8 +143,8 @@ public final class VirtualMutation implements IMutation
      * Accord doesn't support reading/writing virtual tables yet so updating them non-transactionally is always safe
      */
     @Override
-    public boolean allowsPotentialTransactionConflicts()
+    public PotentialTxnConflicts potentialTxnConflicts()
     {
-        return true;
+        return PotentialTxnConflicts.ALLOW;
     }
 }

src/java/org/apache/cassandra/dht/Murmur3Partitioner.java

@@ -258,6 +258,7 @@ public class Murmur3Partitioner implements IPartitioner
             // CASSANDRA-17109 Added the below checks, but paxos tests were not updated, rather than fix
             // the paxos tests, disabling the checks for now.  The current paxos tests bias twards MIN but
             // not for MAX, which makes the test very flaky as when MAX is generated the test fails...
+            // TODO (mustfix): This was done as part of CEP-15 and needs to be added back
 //            if (token == MAXIMUM)
 //                throw new IllegalArgumentException("Cannot increase above MAXIMUM");
 

src/java/org/apache/cassandra/dht/NormalizedRanges.java

@@ -136,6 +136,9 @@ public class NormalizedRanges<T extends RingPosition<T>> extends AbstractList<Ra
 
     public NormalizedRanges<T> subtract(NormalizedRanges<T> b)
     {
+        if (b.isEmpty())
+            return this;
+
         if (b.size() == 1 && b.get(0).isFull())
             return NormalizedRanges.empty();
 
@@ -199,10 +202,9 @@ public class NormalizedRanges<T extends RingPosition<T>> extends AbstractList<Ra
     }
 
     @VisibleForTesting
-    public NormalizedRanges<T> invert()
+    NormalizedRanges<T> invert()
     {
-        if (isEmpty())
-            return this;
+        checkState(!isEmpty());
 
         List<Range<T>> result = new ArrayList<>(size() + 2);
         T minValue = get(0).left.minValue();

src/java/org/apache/cassandra/dht/Range.java

@@ -35,6 +35,7 @@ import org.apache.commons.lang3.ObjectUtils;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Token.KeyBound;
 import org.apache.cassandra.dht.Token.TokenFactory;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -44,6 +45,8 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.Pair;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static java.util.Collections.emptyList;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANGE_EXPENSIVE_CHECKS;
 
@@ -776,4 +779,88 @@ public class Range<T extends RingPosition<T>> extends AbstractBounds<T> implemen
             return tokenSerializer.serializedSize(t, SERDE_VERSION);
         }
     }
+
+    /**
+     * Returns a Pair containing the intersection (or null) and the remainder of the bounds that is to the right of the
+     * range, the remainder to the left is discarded since it is assumed if you are checking for intersection of multiple ranges
+     * the ranges are being checked in order.
+     */
+    public static Pair<AbstractBounds<PartitionPosition>, AbstractBounds<PartitionPosition>> intersectionAndRemainder(AbstractBounds<PartitionPosition> bounds, org.apache.cassandra.dht.Range<Token> range)
+    {
+        checkArgument((bounds.inclusiveRight() && bounds.inclusiveLeft()) || (bounds.left.compareTo(bounds.right) < 0 || bounds.right.isMinimum()), "Wrap around not handled");
+        boolean boundsInclusiveLeft = bounds.inclusiveLeft() || (bounds.left.getClass() == KeyBound.class && ((KeyBound)bounds.left).isMinimumBound);
+        boolean boundsInclusiveRight = bounds.inclusiveRight() || (bounds.right.getClass() == KeyBound.class && !((KeyBound)bounds.right).isMinimumBound);
+        Token boundsLeft = bounds.left.getToken();
+        Token boundsRight = bounds.right.getToken();
+        Token rangeLeft = range.left;
+        Token rangeRight = range.right;
+        checkState(rangeLeft.compareTo(rangeRight) < 0 || rangeRight.isMinimum(), "Wrap around is not handled");
+
+        // Completely before
+        int rightLeftCmp = boundsRight.compareTo(rangeLeft);
+        // Nothing is > min on the right
+        if (boundsRight.isMinimum())
+            rightLeftCmp = 1;
+        // Range left is not inclusive, doesn't matter whether the bound is inclusive/exclusive left
+        rightLeftCmp = rightLeftCmp == 0 ? -1 : rightLeftCmp;
+        if (rightLeftCmp < 0)
+            return Pair.create(null, null);
+
+        // Completely after
+        int leftRightCmp = boundsLeft.compareTo(rangeRight);
+        // Nothing is > min on the right
+        if (rangeRight.isMinimum())
+            leftRightCmp = -1;
+        // Fixed mismatched inclusivity
+        leftRightCmp = leftRightCmp == 0 && !boundsInclusiveLeft ? 1 : leftRightCmp;
+        if (leftRightCmp > 0)
+            return Pair.create(null, bounds);
+
+        int rightRightCmp = boundsRight.compareTo(rangeRight);
+        // min on the right is > than everything
+        if (rangeRight.isMinimum() && boundsRight.isMinimum())
+            rightRightCmp = 0;
+        else if (boundsRight.isMinimum())
+            rightRightCmp = 1;
+        else if (rangeRight.isMinimum())
+            rightRightCmp = -1;
+        // Fixed mismatched inclusivity
+        rightRightCmp = rightRightCmp == 0 && !boundsInclusiveRight ? -1 : rightRightCmp;
+
+        int leftLeftCmp = boundsLeft.compareTo(rangeLeft);
+        // Range left is not inclusive, doesn't matter whether the bound is inclusive/exclusive left
+        leftLeftCmp = leftLeftCmp == 0 ? -1 : leftLeftCmp;
+
+        // Fully contained
+        if (leftLeftCmp > 0 && rightRightCmp <= 0)
+            return Pair.create(bounds, null);
+        // Split by the right bound of the range (rightRightCmp is implicitly > 0 given the preceding condition)
+        else if (leftLeftCmp >= 0)
+            return bounds.split(rangeRight.maxKeyBound());
+        // Intersects but has some portion that needs to be discarded first
+        else
+        {
+            // Remove everything before the intersection
+            Pair<AbstractBounds<PartitionPosition>, AbstractBounds<PartitionPosition>> split = bounds.split(rangeLeft.maxKeyBound());
+            AbstractBounds<PartitionPosition> intersectionAndRemainder = bounds;
+            if (split != null)
+                intersectionAndRemainder = split.right;
+            // There is a remainder
+            if (rightRightCmp > 0)
+                return intersectionAndRemainder.split(rangeRight.maxKeyBound());
+            // There is no remainder everything that
+            return Pair.create(intersectionAndRemainder, null);
+        }
+    }
+
+    public static int compareRightToken(Token a, Token b)
+    {
+        if (a.isMinimum() && b.isMinimum())
+            return 0;
+        if (a.isMinimum())
+            return 1;
+        if (b.isMinimum())
+            return 0;
+        return a.compareTo(b);
+    }
 }

src/java/org/apache/cassandra/hints/HintsDispatcher.java

@@ -42,6 +42,8 @@ import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.exceptions.RequestFailure;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
+import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.InetAddressAndPort;
@@ -248,7 +250,9 @@ final class HintsDispatcher implements AutoCloseable
                 failedRetryDifferentSystem = true;
         }
 
-        if (failures > 0 || timeouts > 0 || failedRetryDifferentSystem)
+        // The batchlog Accord hints need to return abort if any hint needs to be retried and retry the whole page
+        // since we don't want hints to ping pong back and forth via hintsNeedingRehinting
+        if (failures > 0 || timeouts > 0 || failedRetryDifferentSystem || (isBatchLogHints && retryDifferentSystem > 0))
         {
             HintDiagnostics.pageFailureResult(this, success, failures, timeouts, retryDifferentSystem);
             return Action.ABORT;
@@ -534,16 +538,26 @@ final class HintsDispatcher implements AutoCloseable
             try
             {
                 IAccordService accord = AccordService.instance();
-                TxnResult.Kind kind = accord.getTxnResult(accordTxnResult, true, null, requestTime).kind();
+                TxnResult.Kind kind = accord.getTxnResult(accordTxnResult).kind();
                 if (kind == retry_new_protocol)
                     accordOutcome = RETRY_DIFFERENT_SYSTEM;
                 else
                     accordOutcome = SUCCESS;
             }
+            catch (WriteTimeoutException | WriteFailureException | RetryOnDifferentSystemException e)
+            {
+                if (e instanceof RetryOnDifferentSystemException)
+                    accordOutcome = RETRY_DIFFERENT_SYSTEM;
+                else
+                    accordOutcome = TIMEOUT;
+                String msg = "Accord hint delivery transaction failed retriably";
+                if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
+                    logger.error(msg, e);
+            }
             catch (Exception e)
             {
-                accordOutcome = e instanceof WriteTimeoutException ? TIMEOUT : FAILURE;
-                String msg = "Accord hint delivery transaction failed";
+                accordOutcome = FAILURE;
+                String msg = "Accord hint delivery transaction failed permanently";
                 if (noSpamLogger.getStatement(msg).shouldLog(Clock.Global.nanoTime()))
                     logger.error(msg, e);
             }

src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java

@@ -21,17 +21,29 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.internal.CassandraIndexSearcher;
 import org.apache.cassandra.index.internal.IndexEntry;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.btree.BTreeSet;
 
 
@@ -159,7 +171,7 @@ public class CompositesSearcher extends CassandraIndexSearcher
                                                                     DataLimits.NONE,
                                                                     partitionKey,
                                                                     filter,
-                                                                    null);
+                                                                    (Index.QueryPlan) null);
                     }
 
                     // by the next caller of next, or through closing this iterator is this come before.

src/java/org/apache/cassandra/index/internal/keys/KeysSearcher.java

@@ -22,12 +22,22 @@ import java.nio.ByteBuffer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Rows;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.internal.CassandraIndex;
 import org.apache.cassandra.index.internal.CassandraIndexSearcher;
 import org.apache.cassandra.schema.TableMetadata;
@@ -91,7 +101,7 @@ public class KeysSearcher extends CassandraIndexSearcher
                                                                                            DataLimits.NONE,
                                                                                            key,
                                                                                            command.clusteringIndexFilter(key),
-                                                                                           null);
+                                                                                           (Index.QueryPlan) null);
 
                                                   // Otherwise, we close right away if empty, and if it's assigned to next it will be called either
                                                   // by the next caller of next, or through closing this iterator is this come before.

src/java/org/apache/cassandra/index/sai/utils/IndexTermType.java

@@ -788,7 +788,7 @@ public class IndexTermType
                         return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
                 }
             default:
-                throw new IllegalArgumentException("Unsupported collection type: " + collection.kind + "; index type: " + indexType.name());
+                throw new IllegalArgumentException("Unsupported collection type: " + collection.kind);
         }
     }
 

src/java/org/apache/cassandra/metrics/TableMetrics.java

@@ -820,7 +820,7 @@ public class TableMetrics
         accordRepairUnexpectedFailures = createTableMeter("AccordRepairUnexpectedFailures", cfs.keyspace.metric.rangeMigrationUnexpectedFailures);
         accordRepairDependencyLimitFailures = createTableMeter("AccordRepairDependencyLimitFaiures", cfs.keyspace.metric.rangeMigrationDependencyLimitFailures);
         mutationsRejectedOnWrongSystem = createTableMeter("MutationsRejectedOnWrongSystem", cfs.keyspace.metric.mutationsRejectedOnWrongSystem);
-        readsRejectedOnWrongSystem = createTableMeter("ReadsRejectedOnWrongSystem", cfs.keyspace.metric.mutationsRejectedOnWrongSystem);
+        readsRejectedOnWrongSystem = createTableMeter("ReadsRejectedOnWrongSystem", cfs.keyspace.metric.readsRejectedOnWrongSystem);
 
         repairsStarted = createTableCounter("RepairJobsStarted");
         repairsCompleted = createTableCounter("RepairJobsCompleted");

src/java/org/apache/cassandra/schema/TableParams.java

@@ -27,6 +27,7 @@ import com.google.common.base.Objects;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 
+import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.cql3.Attributes;
 import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -244,6 +245,9 @@ public final class TableParams
 
         if (cdc && memtable.factory().writesShouldSkipCommitLog())
             fail("CDC cannot work if writes skip the commit log. Check your memtable configuration.");
+
+        if (transactionalMode.isTestMode() && !CassandraRelevantProperties.ACCORD_ALLOW_TEST_MODES.getBoolean())
+            fail("Transactional mode " + transactionalMode + " can't be used if " + CassandraRelevantProperties.ACCORD_ALLOW_TEST_MODES.getKey() + " is not set");
     }
 
     private static void fail(String format, Object... args)

src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java

@@ -167,7 +167,8 @@ public abstract class AbstractWriteResponseHandler<T> implements RequestCallback
                 // Retrying on the correct system might make this write succeed
                 if (transactionRetryErrors > 0)
                     throw new RetryOnDifferentSystemException();
-                throw new CoordinatorBehindException("Write request failed due to coordinator behind");
+                if (coordinatorBehindErrors > 0)
+                    throw new CoordinatorBehindException("Write request failed due to coordinator behind");
             }
 
             throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, getFailureReasonByEndpointMap());

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -22,8 +22,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -43,8 +45,10 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.cache.CacheLoader;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -84,7 +88,6 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.db.view.ViewUtils;
 import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.CasWriteTimeoutException;
 import org.apache.cassandra.exceptions.CasWriteUnknownResultException;
@@ -138,9 +141,7 @@ import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;
 import org.apache.cassandra.service.accord.txn.TxnDataValue;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
-import org.apache.cassandra.service.accord.txn.TxnRangeRead;
 import org.apache.cassandra.service.accord.txn.TxnRangeReadResult;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnResult;
@@ -148,6 +149,7 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitConsumer;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper.SplitMutations;
 import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
+import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.SplitReads;
 import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
 import org.apache.cassandra.service.paxos.Ballot;
 import org.apache.cassandra.service.paxos.Commit;
@@ -181,11 +183,11 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static accord.primitives.Txn.Kind.Read;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.concat;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.db.ConsistencyLevel.SERIAL;
+import static org.apache.cassandra.db.partitions.PartitionIterators.singletonIterator;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casReadMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.casWriteMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
@@ -213,6 +215,7 @@ import static org.apache.cassandra.service.consensus.migration.ConsensusMigratio
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.getTableMetadata;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.shouldReadEphemerally;
+import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.splitReadsIntoAccordAndNormal;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.GLOBAL;
 import static org.apache.cassandra.service.paxos.Ballot.Flag.LOCAL;
 import static org.apache.cassandra.service.paxos.BallotGenerator.Global.nextBallot;
@@ -1272,7 +1275,7 @@ public class StorageProxy implements StorageProxyMBean
                 {
                     writeMetrics.retryDifferentSystem.mark();
                     writeMetricsForLevel(consistencyLevel).retryDifferentSystem.mark();
-                    logger.debug("Retrying mutations on different system because some mutations were misrouted");
+                    logger.debug("Retrying mutations on different system because some mutations were misrouted according to Cassandra");
                     Tracing.trace("Got {} from normal mutations, will retry", e);
                     continue;
                 }
@@ -1296,9 +1299,13 @@ public class StorageProxy implements StorageProxyMBean
                     if (accordResult != null)
                     {
                         IAccordService accord = AccordService.instance();
-                        TxnResult.Kind kind = accord.getTxnResult(accordResult, true, consistencyLevel, requestTime).kind();
-                        if (kind == retry_new_protocol)
+                        TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
+                        if (kind == retry_new_protocol && failure == null)
+                        {
+                            Tracing.trace("Accord returned retry new protocol");
+                            logger.debug("Retrying mutations on different system because some mutations were misrouted according to Accord");
                             continue;
+                        }
                         Tracing.trace("Successfully wrote Accord mutations");
                     }
                 }
@@ -1400,9 +1407,9 @@ public class StorageProxy implements StorageProxyMBean
             boolean wroteToBatchLog = false;
             while (true)
             {
+                ClusterMetadata cm = ClusterMetadata.current();
                 // In case we hit an error in before/during splitting
                 attributeNonAccordLatency = true;
-                ClusterMetadata cm = ClusterMetadata.current();
                 List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
                 List<Mutation> accordMutations = new ArrayList<>(mutations.size());
                 BatchlogCleanup cleanup = new BatchlogCleanup(() -> asyncRemoveFromBatchlog(batchlogReplicaPlan, batchUUID, requestTime));
@@ -1506,7 +1513,7 @@ public class StorageProxy implements StorageProxyMBean
                     if (accordResult != null)
                     {
                         IAccordService accord = AccordService.instance();
-                        TxnResult.Kind kind = accord.getTxnResult(accordResult, true, consistencyLevel, requestTime).kind();
+                        TxnResult.Kind kind = accord.getTxnResult(accordResult).kind();
                         if (kind == retry_new_protocol && failure == null)
                             continue;
                         Tracing.trace("Successfully wrote Accord mutations");
@@ -2121,7 +2128,7 @@ public class StorageProxy implements StorageProxyMBean
 
         return consistencyLevel.isSerialConsistency()
              ? readWithConsensus(group, consistencyLevel, requestTime)
-             : readRegular(group, consistencyLevel, requestTime);
+             : dispatchReadWithRetryOnDifferentSystem(group, consistencyLevel, ReadCoordinator.DEFAULT, requestTime);
     }
 
     public static boolean hasJoined()
@@ -2176,45 +2183,44 @@ public class StorageProxy implements StorageProxyMBean
         return lastResult.serialReadResult;
     }
 
-    private static ConsistencyLevel consistencyLevelForAccordRead(ClusterMetadata cm, SinglePartitionReadCommand.Group group, @Nullable ConsistencyLevel consistencyLevel)
+    public static ConsistencyLevel consistencyLevelForAccordRead(ClusterMetadata cm, TableId tableId, SinglePartitionReadCommand.Group group, @Nullable ConsistencyLevel consistencyLevel)
     {
         // Null means no specific consistency behavior is required from Accord, it's functionally similar to
         // reading at ONE if you are reading data that wasn't written via Accord
         if (consistencyLevel == null)
             return null;
 
-        TableId tableId = group.queries.get(0).metadata().id;
         TableParams tableParams = getTableMetadata(cm, tableId).params;
         TransactionalMode mode = tableParams.transactionalMode;
         TransactionalMigrationFromMode migrationFromMode = tableParams.transactionalMigrationFrom;
         for (SinglePartitionReadCommand command : group.queries)
         {
-            // readCLForStrategy should return either null or the supplied consistency level
+            // readCLForMode should return either null or the supplied consistency level
             // in which case we will read everything at that CL since Accord doesn't support per table
             // read consistency
-            ConsistencyLevel commitCL = mode.readCLForStrategy(migrationFromMode, consistencyLevel, cm, tableId, command.partitionKey().getToken());
-            if (commitCL != null)
-                return commitCL;
+            ConsistencyLevel readCL = mode.readCLForMode(migrationFromMode, consistencyLevel, cm, tableId, command.partitionKey().getToken());
+            if (readCL != null)
+                return readCL;
         }
         return null;
     }
 
-    public static AsyncTxnResult readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, List<AbstractBounds<PartitionPosition>> ranges, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)    {
+    public static AsyncTxnResult readWithAccord(ClusterMetadata cm, PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> range, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    {
         if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel))
             throw new InvalidRequestException(consistencyLevel + " is not supported by Accord");
 
         TableMetadata tableMetadata = getTableMetadata(cm, command.metadata().id);
         TableParams tableParams = tableMetadata.params;
-        Range<Token> readRange = new Range<>(command.dataRange().startKey().getToken(), command.dataRange().stopKey().getToken());
-        consistencyLevel = tableParams.transactionalMode.readCLForStrategy(tableParams.transactionalMigrationFrom, consistencyLevel, cm, tableMetadata.id, readRange);
-        TxnRead read = new TxnRangeRead(command, ranges, consistencyLevel);
+        consistencyLevel = tableParams.transactionalMode.readCLForMode(tableParams.transactionalMigrationFrom, consistencyLevel, cm, tableMetadata.id, command.dataRange().keyRange());
+        TxnRead read = TxnRead.createRangeRead(command, range, consistencyLevel);
         Txn.Kind kind = shouldReadEphemerally(read.keys(), tableParams, Read);
         Txn txn = new Txn.InMemory(kind, read.keys(), read, TxnQuery.RANGE_QUERY, null);
         IAccordService accordService = AccordService.instance();
         return accordService.coordinateAsync(tableMetadata.epoch.getEpoch(), txn, consistencyLevel, requestTime);
     }
 
-    private static ConsensusAttemptResult readWithAccord(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    private static AsyncTxnResult readWithAccordAsync(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     {
         if (consistencyLevel != null && !IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS.contains(consistencyLevel))
             throw new InvalidRequestException(consistencyLevel + " is not supported by Accord");
@@ -2223,20 +2229,25 @@ public class StorageProxy implements StorageProxyMBean
         // level since Accord will manage reading safely
         TableMetadata tableMetadata = getTableMetadata(cm, group.metadata().id);
         TableParams tableParams = tableMetadata.params;
-        consistencyLevel = consistencyLevelForAccordRead(cm, group, consistencyLevel);
-        TxnKeyRead read = TxnKeyRead.createSerialRead(group.queries, consistencyLevel);
+        consistencyLevel = consistencyLevelForAccordRead(cm, group.queries.get(0).metadata().id, group, consistencyLevel);
+        TxnRead read = TxnRead.createSerialRead(group.queries, consistencyLevel);
         Txn.Kind kind = shouldReadEphemerally(read.keys(), tableParams, Read);
         Txn txn = new Txn.InMemory(kind, read.keys(), read, TxnQuery.ALL, null);
-        AsyncTxnResult asyncTxnResult = AccordService.instance().coordinateAsync(tableMetadata.epoch.getEpoch(), txn, consistencyLevel, requestTime);
-        return getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, group.queries.size(), index -> group.queries.get(index).isReversed(), consistencyLevel, requestTime);
+        return AccordService.instance().coordinateAsync(tableMetadata.epoch.getEpoch(), txn, consistencyLevel, requestTime);
+    }
+
+    private static ConsensusAttemptResult readWithAccord(ClusterMetadata cm, SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    {
+        AsyncTxnResult asyncTxnResult = readWithAccordAsync(cm, group, consistencyLevel, requestTime);
+        return getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, group.queries.size(), index -> group.queries.get(index).isReversed());
     }
 
     /*
      * Used for both the SERIAL and non-SERIAL read path into Accord
      */
-    public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(AsyncTxnResult asyncTxnResult, int numQueries, IntPredicate isQueryReversed, ConsistencyLevel cl, Dispatcher.RequestTime requestTime)
+    public static ConsensusAttemptResult getConsensusAttemptResultFromAsyncTxnResult(AsyncTxnResult asyncTxnResult, int numQueries, IntPredicate isQueryReversed)
     {
-        TxnResult txnResult = AccordService.instance().getTxnResult(asyncTxnResult, false, cl, requestTime);
+        TxnResult txnResult = AccordService.instance().getTxnResult(asyncTxnResult);
         // TODO (required): Converge on a single approach to RETRY_NEW_PROTOCOL, this works for now because reads don't support it anyways
         if (txnResult.kind() == retry_new_protocol)
             return RETRY_NEW_PROTOCOL;
@@ -2251,7 +2262,7 @@ public class StorageProxy implements StorageProxyMBean
         else if (data.size() == 1)
         {
             TxnDataKeyValue value = ((TxnDataKeyValue)data.values().iterator().next());
-            return serialReadResult(PartitionIterators.singletonIterator(value.rowIterator(isQueryReversed.test(0))));
+            return serialReadResult(singletonIterator(value.rowIterator(isQueryReversed.test(0))));
         }
         else
         {
@@ -2263,7 +2274,7 @@ public class StorageProxy implements StorageProxyMBean
             {
                 int queryIndex = e.getKey();
                 TxnDataKeyValue value = ((TxnDataKeyValue)e.getValue());
-                partitionIterators.set(queryIndex, PartitionIterators.singletonIterator(value.rowIterator(isQueryReversed.test(queryIndex))));
+                partitionIterators.set(queryIndex, singletonIterator(value.rowIterator(isQueryReversed.test(queryIndex))));
             }
             return serialReadResult(partitionIterators.size() == 1 ? partitionIterators.get(0) : PartitionIterators.concat(partitionIterators));
         }
@@ -2364,45 +2375,124 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
+    public static PartitionIterator dispatchReadWithRetryOnDifferentSystem(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ReadCoordinator coordinator, Dispatcher.RequestTime requestTime)
+    throws UnavailableException, ReadFailureException, ReadTimeoutException
+    {
+        while (true)
+        {
+            ClusterMetadata cm = ClusterMetadata.current();
+            try
+            {
+                SplitReads splitReads = splitReadsIntoAccordAndNormal(cm, group, coordinator, requestTime);
+                SinglePartitionReadCommand.Group accordReads = splitReads.accordReads;
+                AsyncTxnResult accordResult = accordReads != null ? readWithAccordAsync(cm, accordReads, consistencyLevel, requestTime) : null;
+                SinglePartitionReadCommand.Group normalReads = splitReads.normalReads;
+                Tracing.trace("Split reads into Accord {} and normal {}", accordReads, normalReads);
+
+                Throwable failure = null;
+                PartitionIterator normalPartitions = null;
+                try
+                {
+                    if (normalReads != null)
+                    {
+                        normalPartitions = readRegular(normalReads, consistencyLevel, coordinator, requestTime);
+                        Tracing.trace("Successfully executed normal reads");
+                    }
+                }
+                catch (RetryOnDifferentSystemException e)
+                {
+                    readMetrics.retryDifferentSystem.mark();
+                    readMetricsForLevel(consistencyLevel).retryDifferentSystem.mark();
+                    logger.debug("Retrying reads on different system because some reads were misrouted according to Accord");
+                    Tracing.trace("Got {} from normal reads, will retry", e);
+                    continue;
+                }
+                catch (CoordinatorBehindException e)
+                {
+                    readMetrics.retryCoordinatorBehind.mark();
+                    readMetricsForLevel(consistencyLevel).retryCoordinatorBehind.mark();
+                    logger.debug("Retrying reads now that coordinator has caught up to cluster metadata");
+                    Tracing.trace("Got {} from normal reads, will retry", e);
+                    continue;
+                }
+                catch (Exception e)
+                {
+                    failure = Throwables.merge(failure, e);
+                }
+
+                // Check if the Accord reads succeeded asynchronously
+                PartitionIterator accordPartitions = null;
+                try
+                {
+                    if (accordResult != null)
+                    {
+                        ConsensusAttemptResult consensusResult = getConsensusAttemptResultFromAsyncTxnResult(accordResult, accordReads.queries.size(), index -> group.queries.get(index).isReversed());
+                        if (consensusResult == RETRY_NEW_PROTOCOL)
+                        {
+                            readMetrics.retryDifferentSystem.mark();
+                            readMetricsForLevel(consistencyLevel).retryDifferentSystem.mark();
+                            Tracing.trace("Accord returned retry new protocol");
+                            logger.debug("Retrying reads on different system because some reads were misrouted according to Accord");
+                            continue;
+                        }
+                        Tracing.trace("Successfully executed Accord reads");
+                        accordPartitions = consensusResult.serialReadResult;
+                    }
+                }
+                catch (Exception e)
+                {
+                    failure = Throwables.merge(failure, e);
+                }
+
+                if (failure != null)
+                    throw unchecked(failure);
+
+                PartitionIterator resultIterator = null;
+                if (normalPartitions != null && (accordPartitions == null || !accordPartitions.hasNext()))
+                    resultIterator = normalPartitions;
+                else if ((normalPartitions == null || !normalPartitions.hasNext()) && accordPartitions != null)
+                    resultIterator = accordPartitions;
+                else
+                {
+                    // Merge into partition key order
+                    List<PartitionIterator> partitions = new ArrayList<>(group.queries.size());
+                    Iterator<RowIterator> mergeIterator = Iterators.mergeSorted(ImmutableList.of(normalPartitions, accordPartitions), Comparator.comparing(RowIterator::partitionKey));
+                    while (mergeIterator.hasNext())
+                    {
+                        partitions.add(singletonIterator(mergeIterator.next()));
+                    }
+                    resultIterator = PartitionIterators.concat(partitions);}
+                return maybeEnforceLimits(resultIterator, group);
+            }
+            catch (Exception t)
+            {
+                // Unexpected error so it would be helpful to have details
+                Tracing.trace("{}", getStackTraceAsToString(t));
+                throw t;
+            }
+        }
+    }
+
+    public static PartitionIterator maybeEnforceLimits(PartitionIterator iterator, SinglePartitionReadCommand.Group group)
+    {
+        // Note that the only difference between the command in a group must be the partition key on which
+        // they applied.
+        boolean enforceStrictLiveness = group.queries.get(0).metadata().enforceStrictLiveness();
+        // If we have more than one command, then despite each read command honoring the limit, the total result
+        // might not honor it and so we should enforce it
+        if (group.queries.size() > 1)
+            return group.limits().filter(iterator, group.nowInSec(), group.selectsFullPartition(), enforceStrictLiveness);
+        return iterator;
+    }
+
     @SuppressWarnings("resource")
-    public static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ReadCoordinator coordinator, Dispatcher.RequestTime requestTime)
+    private static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, ReadCoordinator coordinator, Dispatcher.RequestTime requestTime)
     throws UnavailableException, ReadFailureException, ReadTimeoutException
     {
         long start = nanoTime();
         try
         {
-            ClusterMetadata cm = ClusterMetadata.current();
-            TableId tableId = group.queries.get(0).metadata().id;
-            // Returns null for local tables
-            TableMetadata tableMetadata = getTableMetadata(cm, tableId);
-            if (tableMetadata == null)
-                tableMetadata = Schema.instance.localKeyspaces().getTableOrViewNullable(tableId);
-            TableParams tableParams = tableMetadata.params;
-
-            TransactionalMode transactionalMode = tableParams.transactionalMode;
-//            TransactionalMigrationFromMode transactionalMigrationFromMode = tableParams.transactionalMigrationFrom;
-            // TODO (required): Tests would fail with this and we need to add live migration support anyways so for now allow it
-//            if (transactionalMigrationFromMode != TransactionalMigrationFromMode.none)
-//                throw new UnsupportedOperationException("Live migration is not supported, can't safely read when migrating from " + transactionalMigrationFromMode + " to " + transactionalMode);
-
-            PartitionIterator result;
-            if (transactionalMode.nonSerialReadsThroughAccord && coordinator.isEventuallyConsistent())
-            {
-                ConsensusAttemptResult consensusAttemptResult = readWithAccord(cm, group, consistencyLevel, requestTime);
-                checkState(!consensusAttemptResult.shouldRetryOnNewConsensusProtocol, "Live migration is not supported with non-SERIAL reads yet");
-                result = consensusAttemptResult.serialReadResult;
-            }
-            else
-                result = fetchRows(group.queries, consistencyLevel, coordinator, requestTime);
-
-            // Note that the only difference between the command in a group must be the partition key on which
-            // they applied.
-            boolean enforceStrictLiveness = tableMetadata.enforceStrictLiveness();
-            // If we have more than one command, then despite each read command honoring the limit, the total result
-            // might not honor it and so we should enforce it
-            if (group.queries.size() > 1)
-                result = group.limits().filter(result, group.nowInSec(), group.selectsFullPartition(), enforceStrictLiveness);
-            return result;
+            return fetchRows(group.queries, consistencyLevel, coordinator, requestTime);
         }
         catch (UnavailableException e)
         {
@@ -2444,11 +2534,6 @@ public class StorageProxy implements StorageProxyMBean
         }
     }
 
-    public static PartitionIterator readRegular(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
-    {
-        return readRegular(group, consistencyLevel, ReadCoordinator.DEFAULT, requestTime);
-    }
-
     public static void recordReadRegularAbort(ConsistencyLevel consistencyLevel, Throwable cause)
     {
         readMetrics.markAbort(cause);
@@ -2493,8 +2578,11 @@ public class StorageProxy implements StorageProxyMBean
      * 3. Wait for a response from R replicas
      * 4. If the digests (if any) match the data return the data
      * 5. else carry out read repair by getting data from all the nodes.
+     *
+     * This should not be called directly because it bypasses statistics and error handling. It is public
+     * so it can be used by Accord to fetch rows and the statistics will be tracked by Accord.
      */
-    private static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands,
+    public static PartitionIterator fetchRows(List<SinglePartitionReadCommand> commands,
                                                ConsistencyLevel consistencyLevel,
                                                ReadCoordinator coordinator,
                                                Dispatcher.RequestTime requestTime)

src/java/org/apache/cassandra/service/StorageService.java

@@ -1677,14 +1677,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     }
 
     @Override
-    public void migrateConsensusProtocol(@Nonnull String targetProtocol,
-                                         @Nonnull List<String> keyspaceNames,
+    public void migrateConsensusProtocol(@Nonnull List<String> keyspaceNames,
                                          @Nullable List<String> maybeTableNames,
                                          @Nullable String maybeRangesStr)
     {
-        checkNotNull(targetProtocol, "targetProtocol is null");
         checkArgument(!keyspaceNames.contains(SchemaConstants.METADATA_KEYSPACE_NAME));
-        startMigrationToConsensusProtocol(targetProtocol, keyspaceNames, Optional.ofNullable(maybeTableNames), Optional.ofNullable(maybeRangesStr));
+        startMigrationToConsensusProtocol(keyspaceNames, Optional.ofNullable(maybeTableNames), Optional.ofNullable(maybeRangesStr));
     }
 
     @Override

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -1142,8 +1142,7 @@ public interface StorageServiceMBean extends NotificationEmitter
     public String getBootstrapState();
     void abortBootstrap(String nodeId, String endpoint);
 
-    void migrateConsensusProtocol(@Nonnull String targetProtocol,
-                                  @Nullable List<String> keyspaceNames,
+    void migrateConsensusProtocol(@Nullable List<String> keyspaceNames,
                                   @Nullable List<String> maybeTableNames,
                                   @Nullable String maybeRangesStr);
 

src/java/org/apache/cassandra/service/accord/AccordCallback.java

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.coordinate.Timeout;
-import accord.local.AgentExecutor;
-import accord.messages.Callback;
-import accord.messages.Reply;
-import accord.messages.SafeCallback;
-import org.apache.cassandra.exceptions.RequestFailure;
-import org.apache.cassandra.exceptions.RequestFailureReason;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.RequestCallback;
-
-class AccordCallback<T extends Reply> extends SafeCallback<T> implements RequestCallback<T>
-{
-    private static final Logger logger = LoggerFactory.getLogger(AccordCallback.class);
-    private final AccordEndpointMapper endpointMapper;
-
-    public AccordCallback(AgentExecutor executor, Callback<T> callback, AccordEndpointMapper endpointMapper)
-    {
-        super(executor, callback);
-        this.endpointMapper = endpointMapper;
-    }
-
-    @Override
-    public void onResponse(Message<T> msg)
-    {
-        logger.trace("Received response {} from {}", msg.payload, msg.from());
-        success(endpointMapper.mappedId(msg.from()), msg.payload);
-    }
-
-    private static Throwable convertFailureMessage(RequestFailure failure)
-    {
-        return failure.reason == RequestFailureReason.TIMEOUT ?
-               new Timeout(null, null) :
-               new RuntimeException(failure.failure);
-    }
-
-    @Override
-    public void onFailure(InetAddressAndPort from, RequestFailure failure)
-    {
-        logger.trace("Received failure {} from {} for {}", failure, from, this);
-        // TODO (now): we should distinguish timeout failures with some placeholder Exception
-        failure(endpointMapper.mappedId(from), convertFailureMessage(failure));
-    }
-
-    @Override
-    public boolean trackLatencyForSnitch()
-    {
-        return true;
-    }
-
-    @Override
-    public boolean invokeOnFailure()
-    {
-        return true;
-    }
-}

src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java

@@ -52,13 +52,16 @@ import org.apache.cassandra.repair.SharedContext;
 import org.apache.cassandra.service.accord.AccordKeyspace.EpochDiskState;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.listeners.ChangeListener;
+import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Simulate;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
+import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord;
 import static org.apache.cassandra.utils.Simulate.With.MONITORS;
 
 // TODO: listen to FailureDetector and rearrange fast path accordingly
@@ -344,16 +347,22 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
             }
         }
         reportTopology(topology);
-        if (epochs.lastAcknowledged() >= topology.epoch()) checkIfNodesRemoved(topology);
-        else epochs.acknowledgeFuture(topology.epoch()).addCallback(() -> checkIfNodesRemoved(topology));
+        Set<Node.Id> stillLiveNodes = metadata.directory.states.entrySet()
+                                                               .stream()
+                                                               .filter(e -> e.getValue() != NodeState.LEFT && e.getValue() != NodeState.LEAVING)
+                                                               .map(e -> tcmIdToAccord(e.getKey()))
+                                                               .collect(Collectors.toSet());
+        if (epochs.lastAcknowledged() >= topology.epoch()) checkIfNodesRemoved(topology, stillLiveNodes);
+        else epochs.acknowledgeFuture(topology.epoch()).addCallback(() -> checkIfNodesRemoved(topology, stillLiveNodes));
     }
 
-    private void checkIfNodesRemoved(Topology topology)
+    private void checkIfNodesRemoved(Topology topology, Set<Node.Id> stillLiveNodes)
     {
         if (epochs.minEpoch() == topology.epoch()) return;
         Topology previous = getTopologyForEpoch(topology.epoch() - 1);
         // for all nodes removed, or pending removal, mark them as removed so we don't wait on their replies
-        Sets.SetView<Node.Id> removedNodes = Sets.difference(previous.nodes(), topology.nodes());
+        Set<Node.Id> removedNodes = Sets.difference(previous.nodes(), topology.nodes());
+        removedNodes = Sets.filter(removedNodes, id -> !stillLiveNodes.contains(id));
         // TODO (desired, efficiency): there should be no need to notify every epoch for every removed node
         for (Node.Id removedNode : removedNodes)
         {
@@ -420,29 +429,36 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
     @Override
     protected void fetchTopologyInternal(long epoch)
     {
-        try
-        {
-            Set<InetAddressAndPort> peers = new HashSet<>(ClusterMetadata.current().directory.allJoinedEndpoints());
-            peers.remove(FBUtilities.getBroadcastAddressAndPort());
-            if (peers.isEmpty())
-                return;
-            Topology topology;
-            while ((topology =FetchTopology.fetch(SharedContext.Global.instance, peers, epoch).get()) == null) {}
-            reportTopology(topology);
-        }
-        catch (InterruptedException e)
-        {
-            if (currentEpoch() >= epoch)
-                return;
-            Thread.currentThread().interrupt();
-            throw new UncheckedInterruptedException(e);
-        }
-        catch (Throwable e)
-        {
-            if (currentEpoch() >= epoch)
-                return;
-            throw new RuntimeException(e.getCause());
-        }
+        // It's not safe for this to block on CMS so for now pick a thread pool to handle it
+        Stage.ACCORD_MIGRATION.execute(() -> {
+            if (ClusterMetadata.current().epoch.getEpoch() < epoch)
+                ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(epoch));
+            try
+            {
+                Set<InetAddressAndPort> peers = new HashSet<>(ClusterMetadata.current().directory.allJoinedEndpoints());
+                peers.remove(FBUtilities.getBroadcastAddressAndPort());
+                if (peers.isEmpty())
+                    return;
+                Topology topology;
+                while ((topology = FetchTopology.fetch(SharedContext.Global.instance, peers, epoch).get()) == null)
+                {
+                }
+                reportTopology(topology);
+            }
+            catch (InterruptedException e)
+            {
+                if (currentEpoch() >= epoch)
+                    return;
+                Thread.currentThread().interrupt();
+                throw new UncheckedInterruptedException(e);
+            }
+            catch (Throwable e)
+            {
+                if (currentEpoch() >= epoch)
+                    return;
+                throw new RuntimeException(e.getCause());
+            }
+        });
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java

@@ -50,6 +50,7 @@ import accord.primitives.Ranges;
 import accord.primitives.Routable.Domain;
 import accord.primitives.RoutingKeys;
 import accord.primitives.SaveStatus;
+import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Status;
 import accord.primitives.Timestamp;
@@ -94,6 +95,16 @@ public class AccordObjectSizes
         return EMPTY_RANGE_SIZE + key(range.start()) + key(range.end());
     }
 
+    public static long seekable(Seekable seekable)
+    {
+        switch (seekable.domain())
+        {
+            default: throw new AssertionError();
+            case Key: return key((Key) seekable);
+            case Range: return range((Range) seekable);
+        }
+    }
+
     private static final long EMPTY_RANGES_SIZE = measure(Ranges.of());
     public static long ranges(Ranges ranges)
     {

src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java

@@ -62,6 +62,7 @@ class AccordResponseVerbHandler<T extends Reply> implements IVerbHandler<T>
         }
 
         Node.Id from = endpointMapper.mappedId(message.from());
+        logger.trace("Receiving {} from {}", message.payload, message.from());
         if (message.isFailureResponse())
         {
             Tracing.trace("Processing failure response from {}", message.from());

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -26,6 +26,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -118,11 +119,11 @@ import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.RequestValidations;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.WriteType;
 import org.apache.cassandra.dht.AccordSplitter;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.RequestTimeoutException;
@@ -156,6 +157,7 @@ import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
 import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
 import org.apache.cassandra.service.accord.interop.AccordInteropAdapter.AccordInteropFactory;
 import org.apache.cassandra.service.accord.repair.RepairSyncPointAdapter;
+import org.apache.cassandra.service.accord.txn.RetryWithNewProtocolResult;
 import org.apache.cassandra.service.accord.txn.TxnResult;
 import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.consensus.migration.TableMigrationState;
@@ -609,12 +611,14 @@ public class AccordService implements IAccordService, Shutdownable
         // Barriers can be needed just because it's an Accord managed range, but it could also be a migration back to Paxos
         // in which case we do want to barrier the migrating/migrated ranges even though the target for the migration is not Accord
         // In either case Accord should be aware of those ranges and not generate a topology mismatch
-        if (tm.params.transactionalMode != TransactionalMode.off || tm.params.transactionalMigrationFrom.from != TransactionalMode.off)
+        if (tm.params.transactionalMode != TransactionalMode.off || tm.params.transactionalMigrationFrom.migratingFromAccord())
         {
             TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id);
             // null is fine could be completely migrated or was always an Accord table on creation
             if (tms == null)
                 return keysOrRanges;
+            // Use migratingAndMigratedRanges (not accordSafeToReadRanges) because barriers are allowed even if Accord can't perform
+            // a read because they are only finishing/recovering existing Accord transactions
             Ranges migratingAndMigratedRanges = AccordTopology.toAccordRanges(tms.tableId, tms.migratingAndMigratedRanges);
             return keysOrRanges.slice(migratingAndMigratedRanges);
         }
@@ -739,6 +743,34 @@ public class AccordService implements IAccordService, Shutdownable
         return node.topology();
     }
 
+    private static Set<TableId> txnDroppedTables(Seekables<?,?> keys)
+    {
+        Set<TableId> tables = new HashSet<>();
+        for (Seekable seekable : keys)
+        {
+            switch (seekable.domain())
+            {
+                default:
+                    throw new IllegalStateException("Unhandled domain " + seekable.domain());
+                case Key:
+                    tables.add(((PartitionKey) seekable).table());
+                    break;
+                case Range:
+                    tables.add(((TokenRange) seekable).table());
+                    break;
+            }
+        }
+
+        Iterator<TableId> tablesIterator = tables.iterator();
+        while (tablesIterator.hasNext())
+        {
+            TableId table = tablesIterator.next();
+            if (Schema.instance.getTableMetadata(table) != null)
+                tablesIterator.remove();
+        }
+        return tables;
+    }
+
     /**
      * Consistency level is just echoed back in timeouts, in the future it may be used for interoperability
      * with non-Accord operations.
@@ -747,7 +779,7 @@ public class AccordService implements IAccordService, Shutdownable
     public @Nonnull TxnResult coordinate(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, @Nonnull Dispatcher.RequestTime requestTime)
     {
         AsyncTxnResult asyncTxnResult = coordinateAsync(minEpoch, txn, consistencyLevel, requestTime);
-        return getTxnResult(asyncTxnResult, txn.isWrite(), consistencyLevel, requestTime);
+        return getTxnResult(asyncTxnResult);
     }
 
     @Override
@@ -769,7 +801,8 @@ public class AccordService implements IAccordService, Shutdownable
         metrics.keySize.update(txn.keys().size());
         long deadlineNanos = requestTime.computeDeadline(DatabaseDescriptor.getTransactionTimeout(NANOSECONDS));
         AsyncResult<Result> asyncResult = node.coordinate(txnId, txn, minEpoch, deadlineNanos);
-        AsyncTxnResult asyncTxnResult = new AsyncTxnResult(txnId);
+        Seekables<?, ?> keys = txn.keys();
+        AsyncTxnResult asyncTxnResult = new AsyncTxnResult(txnId, minEpoch, consistencyLevel, txn.isWrite(), requestTime);
         asyncResult.addCallback((success, failure) -> {
             long durationNanos = nanoTime() - requestTime.startedAtNanos();
             sharedMetrics.addNano(durationNanos);
@@ -786,6 +819,9 @@ public class AccordService implements IAccordService, Shutdownable
                 return;
             }
 
+            sharedMetrics.failures.mark();
+            metrics.failures.mark();
+
             if (cause instanceof Timeout)
             {
                 // Don't mark the metric here, should be done in getTxnResult to ensure it only happens once
@@ -795,7 +831,6 @@ public class AccordService implements IAccordService, Shutdownable
             }
             if (cause instanceof Preempted || cause instanceof Invalidated)
             {
-                sharedMetrics.timeouts.mark();
                 metrics.preempted.mark();
                 //TODO need to improve
                 // Coordinator "could" query the accord state to see whats going on but that doesn't exist yet.
@@ -803,25 +838,39 @@ public class AccordService implements IAccordService, Shutdownable
                 asyncTxnResult.tryFailure(newPreempted(txnId, txn.isWrite(), consistencyLevel));
                 return;
             }
-            sharedMetrics.failures.mark();
+            // TODO (desired): It would be better to check if the topology mismatch is due to Accord ownership changes
+            // or the txn accessing tables that don't exist or something.
             if (cause instanceof TopologyMismatch)
             {
+                // Excluding bugs topology mismatch can occur because a table was dropped in between creating the txn
+                // and executing it.
+                // It could also race with the table stopping/starting being managed by Accord.
+                // The caller can retry if the table indeed exists and is managed by Accord.
+                Set<TableId> txnDroppedTables = txnDroppedTables(keys);
+                Tracing.trace("Accord returned topology mismatch: " + cause.getMessage());
+                logger.debug("Accord returned topology mismatch", cause);
                 metrics.topologyMismatches.mark();
-                asyncTxnResult.tryFailure(RequestValidations.invalidRequest(cause.getMessage()));
+                // Throw IRE in case the caller fails to check if the table still exists
+                if (!txnDroppedTables.isEmpty())
+                {
+                    Tracing.trace("Accord txn uses dropped tables {}", txnDroppedTables);
+                    logger.debug("Accord txn uses dropped tables {}", txnDroppedTables);
+                    asyncTxnResult.setFailure(new InvalidRequestException("Accord transaction uses dropped tables"));
+                }
+                asyncTxnResult.setSuccess(RetryWithNewProtocolResult.instance);
                 return;
             }
-            metrics.failures.mark();
             asyncTxnResult.tryFailure(new RuntimeException(cause));
         });
         return asyncTxnResult;
     }
 
     @Override
-    public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+    public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult)
     {
         ClientRequestMetrics sharedMetrics;
         AccordClientRequestMetrics metrics;
-        if (isWrite)
+        if (asyncTxnResult.isWrite)
         {
             sharedMetrics = ClientRequestsMetricsHolder.writeMetrics;
             metrics = accordWriteMetrics;
@@ -833,7 +882,7 @@ public class AccordService implements IAccordService, Shutdownable
         }
         try
         {
-            long deadlineNanos = requestTime.computeDeadline(DatabaseDescriptor.getTransactionTimeout(NANOSECONDS));
+            long deadlineNanos = asyncTxnResult.requestTime.computeDeadline(DatabaseDescriptor.getTransactionTimeout(NANOSECONDS));
             TxnResult result = asyncTxnResult.get(deadlineNanos - nanoTime(), NANOSECONDS);
             return result;
         }
@@ -851,6 +900,8 @@ public class AccordService implements IAccordService, Shutdownable
             }
             else if (cause instanceof RuntimeException)
                 throw (RuntimeException) cause;
+            else if (cause instanceof InvalidRequestException)
+                throw ((InvalidRequestException)cause);
             else
                 throw new RuntimeException(cause);
         }
@@ -864,7 +915,7 @@ public class AccordService implements IAccordService, Shutdownable
         {
             metrics.timeouts.mark();
             sharedMetrics.timeouts.mark();
-            throw newTimeout(asyncTxnResult.txnId, isWrite, consistencyLevel);
+            throw newTimeout(asyncTxnResult.txnId, asyncTxnResult.isWrite, asyncTxnResult.consistencyLevel);
         }
     }
 

src/java/org/apache/cassandra/service/accord/AccordTopology.java

@@ -18,17 +18,25 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
-import accord.local.Node.Id;
-import accord.primitives.Ranges;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 
+import accord.local.Node.Id;
+import accord.primitives.Ranges;
 import accord.topology.Shard;
 import accord.topology.Topology;
 import accord.utils.Invariants;
@@ -38,7 +46,13 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.*;
+import org.apache.cassandra.schema.Diff;
+import org.apache.cassandra.schema.DistributedSchema;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;

src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java

@@ -27,6 +27,9 @@ import accord.local.Node;
 import accord.messages.Request;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.utils.NoSpamLogger;
 
 public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
@@ -53,6 +56,7 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
         }
 
         logger.trace("Receiving {} from {}", message.payload, message.from());
+
         T request = message.payload;
 
         /*
@@ -62,14 +66,32 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
          */
         Node.Id fromNodeId = endpointMapper.mappedId(message.from());
         long waitForEpoch = request.waitForEpoch();
-
-        if (node.topology().hasAtLeastEpoch(waitForEpoch))
+        ClusterMetadata cm = ClusterMetadata.current();
+        boolean cmUpToDate = ClusterMetadata.current().epoch.getEpoch() >= waitForEpoch;
+        if (node.topology().hasAtLeastEpoch(waitForEpoch) && cmUpToDate)
             request.process(node, fromNodeId, message.header);
         else
-            node.withEpoch(waitForEpoch, (ignored, withEpochFailure) -> {
-                if (withEpochFailure != null)
-                    throw new RuntimeException("Timed out waiting for epoch when processing message from " + fromNodeId + " to " + node + " message " + message, withEpochFailure);
-                request.process(node, fromNodeId, message.header);
-            });
+        {
+            // withEpoch does not reliably ensure that TCM is up to date, if Accord has the topology it won't
+            // wait for TCM to come up to date, so do it here in the verb handler
+            if (!cmUpToDate)
+            {
+                ClusterMetadataService.instance().fetchLogFromPeerOrCMSAsync(cm, message.from(), Epoch.create(waitForEpoch)).addCallback((success, failure) ->
+                    node.withEpoch(waitForEpoch, (ignored, withEpochFailure) -> {
+                        if (withEpochFailure != null)
+                            throw new RuntimeException("Timed out waiting for epoch when processing message from " + fromNodeId + " to " + node + " message " + message, withEpochFailure);
+                        request.process(node, fromNodeId, message.header);
+                    })
+                );
+            }
+            else
+            {
+                node.withEpoch(waitForEpoch, (ignored, withEpochFailure) -> {
+                    if (withEpochFailure != null)
+                        throw new RuntimeException("Timed out waiting for epoch when processing message from " + fromNodeId + " to " + node + " message " + message, withEpochFailure);
+                    request.process(node, fromNodeId, message.header);
+                });
+            }
+        }
     }
 }

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -110,17 +110,27 @@ public interface IAccordService
     class AsyncTxnResult extends AsyncPromise<TxnResult>
     {
         public final @Nonnull TxnId txnId;
+        public final long minEpoch;
+        @Nullable
+        public final ConsistencyLevel consistencyLevel;
+        public final boolean isWrite;
+        public final Dispatcher.RequestTime requestTime;
 
-        public AsyncTxnResult(@Nonnull TxnId txnId)
+        public AsyncTxnResult(@Nonnull TxnId txnId, long minEpoch, @Nullable ConsistencyLevel consistencyLevel, boolean isWrite, @Nonnull Dispatcher.RequestTime requestTime)
         {
             checkNotNull(txnId);
+            checkNotNull(requestTime);
             this.txnId = txnId;
+            this.minEpoch = minEpoch;
+            this.consistencyLevel = consistencyLevel;
+            this.isWrite = isWrite;
+            this.requestTime = requestTime;
         }
     }
 
     @Nonnull
     AsyncTxnResult coordinateAsync(long minEpoch, @Nonnull Txn txn, @Nonnull ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
-    TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, @Nullable ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
+    TxnResult getTxnResult(AsyncTxnResult asyncTxnResult);
 
     long currentEpoch();
 
@@ -236,7 +246,7 @@ public interface IAccordService
         }
 
         @Override
-        public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
+        public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult)
         {
             throw new UnsupportedOperationException("No accord transaction should be executed when accord.enabled = false in cassandra.yaml");
         }
@@ -418,9 +428,9 @@ public interface IAccordService
         }
 
         @Override
-        public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult, boolean isWrite, @Nullable ConsistencyLevel consistencyLevel, RequestTime requestTime)
+        public TxnResult getTxnResult(AsyncTxnResult asyncTxnResult)
         {
-            return delegate.getTxnResult(asyncTxnResult, isWrite, consistencyLevel, requestTime);
+            return delegate.getTxnResult(asyncTxnResult);
         }
 
         @Override

src/java/org/apache/cassandra/service/accord/TokenRange.java

@@ -34,10 +34,14 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
+import org.apache.cassandra.utils.ObjectSizes;
 
 public class TokenRange extends Range.EndInclusive
 {
-    public TokenRange(AccordRoutingKey start, AccordRoutingKey end)
+    public static final long EMPTY_SIZE = ObjectSizes.measure(new TokenRange(SentinelKey.min(TableId.fromLong(0)), SentinelKey.max(TableId.fromLong(0))));
+
+    // Don't make this public use create or createUnsafe
+    private TokenRange(AccordRoutingKey start, AccordRoutingKey end)
     {
         super(start, end);
     }
@@ -72,6 +76,11 @@ public class TokenRange extends Range.EndInclusive
         return  (AccordRoutingKey) super.end();
     }
 
+    public long estimatedSizeOnHeap()
+    {
+        return EMPTY_SIZE + start().estimatedSizeOnHeap() + end().estimatedSizeOnHeap();
+    }
+
     public boolean isFullRange()
     {
         return start().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL && end().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL;
@@ -94,7 +103,12 @@ public class TokenRange extends Range.EndInclusive
         return new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end);
     }
 
-    public org.apache.cassandra.dht.Range<Token> toKeyspaceRange ()
+    /*
+     * This behaves quite incorrectly with MinTokenKey because it loses the inclusivity of MinTokenKey in the conversion.
+     * It's not a problem for cluster metadata and topology, but it's quite wrong for queries that convert from Bounds to
+     * Range.
+     */
+    public org.apache.cassandra.dht.Range<Token> toKeyspaceRange()
     {
         IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
         AccordRoutingKey start = start();
@@ -104,7 +118,6 @@ public class TokenRange extends Range.EndInclusive
         return new org.apache.cassandra.dht.Range<>(left, right);
     }
 
-
     public static final Serializer serializer = new Serializer();
 
     public static final class Serializer implements IVersionedSerializer<TokenRange>

src/java/org/apache/cassandra/service/accord/api/AccordAgent.java

@@ -52,8 +52,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.metrics.AccordMetrics;
 import org.apache.cassandra.net.ResponseContext;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
@@ -174,7 +174,7 @@ public class AccordAgent implements Agent
     @Override
     public Txn emptySystemTxn(Kind kind, Routable.Domain domain)
     {
-        return new Txn.InMemory(kind, domain == Key ? Keys.EMPTY : Ranges.EMPTY, TxnKeyRead.EMPTY, TxnQuery.UNSAFE_EMPTY, null);
+        return new Txn.InMemory(kind, domain == Key ? Keys.EMPTY : Ranges.EMPTY, TxnRead.empty(domain), TxnQuery.UNSAFE_EMPTY, null);
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java

@@ -240,7 +240,7 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         public Range asRange()
         {
             checkState(!isMinMinSentinel, "It might be possible to support converting a minmin sentinel to a range, but it needs to be evaluated in the context where it is failing");
-            return new TokenRange(new SentinelKey(table, isMinSentinel, true), this);
+            return TokenRange.create(new SentinelKey(table, isMinSentinel, true), this);
         }
     }
 
@@ -373,7 +373,7 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
                                       ? new SentinelKey(table, true, false)
                                       : new TokenKey(table, token.decreaseSlightly());
 
-            return new TokenRange(before, this);
+            return TokenRange.create(before, this);
         }
 
         public MinTokenKey(TableId tableId, Token token)

src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java

@@ -107,7 +107,7 @@ public class AccordInteropAdapter extends AbstractTxnAdapter
         if (updateKind != AccordUpdate.Kind.UNRECOVERABLE_REPAIR && (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE || txn.read().keys().isEmpty()))
             return false;
 
-        new AccordInteropExecution(node, txnId, txn, updateKind, route, txn.read().keys().toParticipants(), executeAt, deps, callback, executor, consistencyLevel, endpointMapper)
+        new AccordInteropExecution(node, txnId, txn, updateKind, route, executeAt, deps, callback, executor, consistencyLevel, endpointMapper)
             .start();
         return true;
     }

src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java

@@ -24,7 +24,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 
@@ -80,19 +79,16 @@ import org.apache.cassandra.service.accord.AccordEndpointMapper;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.interop.AccordInteropReadCallback.MaximalCommitSender;
 import org.apache.cassandra.service.accord.txn.AccordUpdate;
 import org.apache.cassandra.service.accord.txn.TxnData;
-import org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind;
 import org.apache.cassandra.service.accord.txn.TxnDataKeyValue;
 import org.apache.cassandra.service.accord.txn.TxnDataRangeValue;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
-import org.apache.cassandra.service.accord.txn.TxnRangeRead;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.UnrecoverableRepairUpdate;
 import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
-import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration;
-import org.apache.cassandra.service.consensus.migration.TableMigrationState;
 import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.Dispatcher;
@@ -151,7 +147,6 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
     private final TxnId txnId;
     private final Txn txn;
     private final FullRoute<?> route;
-    private final Participants<?> readScope;
     private final Timestamp executeAt;
     private final Deps deps;
     private final BiConsumer<? super Result, Throwable> callback;
@@ -169,7 +164,7 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
     private final Set<InetAddressAndPort> contacted;
     private final AccordUpdate.Kind updateKind;
 
-    public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute<?> route, Participants<?> readScope, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback,
+    public AccordInteropExecution(Node node, TxnId txnId, Txn txn, AccordUpdate.Kind updateKind, FullRoute<?> route, Timestamp executeAt, Deps deps, BiConsumer<? super Result, Throwable> callback,
                                   AgentExecutor executor, ConsistencyLevel consistencyLevel, AccordEndpointMapper endpointMapper)
     {
         requireArgument(!txn.read().keys().isEmpty() || updateKind == AccordUpdate.Kind.UNRECOVERABLE_REPAIR);
@@ -177,7 +172,6 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
         this.txnId = txnId;
         this.txn = txn;
         this.route = route;
-        this.readScope = readScope;
         this.executeAt = executeAt;
         this.deps = deps;
         this.callback = callback;
@@ -233,123 +227,125 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
     public void sendReadCommand(Message<ReadCommand> message, InetAddressAndPort to, RequestCallback<ReadResponse> callback)
     {
         Node.Id id = endpointMapper.mappedId(to);
-        SinglePartitionReadCommand command = (SinglePartitionReadCommand) message.payload;
+        // TODO (nicetohave): It would be better to use the re-use the command from the transaction but it's fragile
+        // to try and figure out exactly what changed for things like read repair and short read protection
+        // Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be
         // TODO (required): understand interop and whether StableFastPath is appropriate
-        AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, Kind.StableFastPath, executeAt, txn, deps, route, command);
+        AccordInteropStableThenRead commit = new AccordInteropStableThenRead(id, allTopologies, txnId, Kind.StableFastPath, executeAt, txn, deps, route, message.payload);
         node.send(id, commit, executor, new AccordInteropRead.ReadCallback(id, to, message, callback, this));
     }
 
     @Override
     public void sendReadRepairMutation(Message<Mutation> message, InetAddressAndPort to, RequestCallback<Object> callback)
     {
-        requireArgument(message.payload.allowsPotentialTransactionConflicts());
+        requireArgument(message.payload.potentialTxnConflicts().allowed);
+        requireArgument(message.payload.getTableIds().size() == 1);
         Node.Id id = endpointMapper.mappedId(to);
+        Participants<?> readScope = Participants.singleton(txn.read().keys().domain(), new TokenKey(message.payload.getTableIds().iterator().next(), message.payload.key().getToken()));
         AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), message.payload);
         node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this));
     }
 
-    private List<AsyncChain<Data>> keyReadChains(int nowInSeconds, Dispatcher.RequestTime requestTime)
+    private List<AsyncChain<Data>> readChains(Dispatcher.RequestTime requestTime)
     {
-        TxnKeyRead read = (TxnKeyRead) txn.read();
-        List<AsyncChain<Data>> results = new ArrayList<>();
+        TxnRead read = (TxnRead) txn.read();
         Seekables<?, ?> keys = txn.read().keys();
+        switch (keys.domain())
+        {
+            case Key:
+                return keyReadChains(read, keys, requestTime);
+            case Range:
+                return rangeReadChains(read, keys, requestTime);
+            default:
+                throw new IllegalStateException("Unhandled domain " + keys.domain());
+        }
+    }
+
+    private List<AsyncChain<Data>> keyReadChains(TxnRead read, Seekables<?, ?> keys, Dispatcher.RequestTime requestTime)
+    {
+        ClusterMetadata cm = ClusterMetadata.current();
+        List<AsyncChain<Data>> results = new ArrayList<>();
         keys.forEach(key -> {
-            read.forEachWithKey((PartitionKey) key, fragment -> {
-                SinglePartitionReadCommand command = (SinglePartitionReadCommand) fragment.command();
+                         read.forEachWithKey((PartitionKey) key, fragment -> {
+                             SinglePartitionReadCommand command = (SinglePartitionReadCommand) fragment.command();
 
-                // This should only rarely occur when coordinators start a transaction in a migrating range
-                // because they haven't yet updated their cluster metadata.
-                // It would be harmless to do the read, because it will be rejected in `TxnQuery` anyways,
-                // but it's faster to skip the read
-                TableMigrationState tms = ConsensusTableMigration.getTableMigrationState(command.metadata().id);
-                AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics;
-                if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(command.metadata(), tms, command.partitionKey()))
-                {
-                    metrics.migrationSkippedReads.mark();
-                    results.add(AsyncChains.success(TxnData.emptyPartition(fragment.txnDataName(), command)));
-                    return;
-                }
+                             // This should only rarely occur when coordinators start a transaction in a migrating range
+                             // because they haven't yet updated their cluster metadata.
+                             // It would be harmless to do the read, because it will be rejected in `TxnQuery` anyways,
+                             // but it's faster to skip the read
+                             AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics;
+                             // TODO (required): This doesn't use the metadata from the correct epoch
+                             if (!ConsensusRequestRouter.instance.isKeyManagedByAccordForReadAndWrite(cm, command.metadata().id, command.partitionKey()))
+                             {
+                                 metrics.migrationSkippedReads.mark();
+                                 results.add(AsyncChains.success(TxnData.emptyPartition(fragment.txnDataName(), command)));
+                                 return;
+                             }
+
+                             Group group = Group.one(command);
+                             results.add(AsyncChains.ofCallable(Stage.ACCORD_MIGRATION.executor(), () -> {
+                                 TxnData result = new TxnData();
+                                 // Enforcing limits is redundant since we only have a group of size 1, but checking anyways
+                                 // documents the requirement here
+                                 try (PartitionIterator iterator = StorageProxy.maybeEnforceLimits(StorageProxy.fetchRows(group.queries, consistencyLevel, this, requestTime), group))
+                                 {
+                                     if (iterator.hasNext())
+                                     {
+                                         try (RowIterator partition = iterator.next())
+                                         {
+                                             TxnDataKeyValue value = new TxnDataKeyValue(partition);
+                                             if (value.hasRows() || command.selectsFullPartition())
+                                                 result.put(fragment.txnDataName(), value);
+                                         }
+                                     }
+                                 }
+                                 return result;
+                             }));
+                         });
+
+                     });
+        return results;
+    }
+
+    private List<AsyncChain<Data>> rangeReadChains(TxnRead read, Seekables<?, ?> keys, Dispatcher.RequestTime requestTime)
+    {
+        List<AsyncChain<Data>> results = new ArrayList<>();
+        keys.forEach(key -> {
+            read.forEachWithKey(key, fragment -> {
+                PartitionRangeReadCommand command = ((PartitionRangeReadCommand) fragment.command()).withTxnReadName(fragment.txnDataName());
+
+                // TODO (required): To make migration work we need to validate that the range is all on Accord
 
-                Group group = Group.one(command.withNowInSec(nowInSeconds));
                 results.add(AsyncChains.ofCallable(Stage.ACCORD_MIGRATION.executor(), () -> {
                     TxnData result = new TxnData();
-                    try (PartitionIterator iterator = StorageProxy.readRegular(group, consistencyLevel, this, requestTime))
+                    try (PartitionIterator iterator = StorageProxy.getRangeSlice(command, consistencyLevel, this, requestTime))
                     {
-                        if (iterator.hasNext())
+                        TxnDataRangeValue value = new TxnDataRangeValue();
+                        while (iterator.hasNext())
                         {
                             try (RowIterator partition = iterator.next())
                             {
-                                TxnDataKeyValue value = new TxnDataKeyValue(partition);
-                                if (value.hasRows() || command.selectsFullPartition())
-                                    result.put(fragment.txnDataName(), value);
+                                FilteredPartition filtered = FilteredPartition.create(partition);
+                                if (filtered.hasRows() || command.selectsFullPartition())
+                                    value.add(filtered);
                             }
                         }
+                        result.put(fragment.txnDataName(), value);
                     }
                     return result;
                 }));
             });
-        });
-        return results;
-    }
 
-    private List<AsyncChain<Data>> rangeReadChains(long nowInSeconds, Dispatcher.RequestTime requestTime)
-    {
-        TxnRangeRead read = (TxnRangeRead) txn.read();
-        Seekables<?, ?> keys = txn.read().keys();
-        List<AsyncChain<Data>> results = new ArrayList<>();
-        keys.forEach(key -> {
-            TokenRange range = (TokenRange)key;
-            PartitionRangeReadCommand command = read.commandForSubrange(range, nowInSeconds);
-
-            // This should only rarely occur when coordinators start a transaction in a migrating range
-            // because they haven't yet updated their cluster metadata.
-            // It would be harmless to do the read, because it will be rejected in `TxnQuery` anyways,
-            // but it's faster to skip the read
-            // TODO (required): To make migration work we need to validate that the range is all on Accord
-            // if any part isn't we should reject the read
-//                TableMigrationState tms = ConsensusTableMigration.getTableMigrationState(command.metadata().id);
-//                AccordClientRequestMetrics metrics = txn.kind().isWrite() ? accordWriteMetrics : accordReadMetrics;
-//                if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(command.metadata(), tms, command.partitionKey()))
-//                {
-//                    metrics.migrationSkippedReads.mark();
-//                    results.add(AsyncChains.success(TxnData.emptyPartition(fragment.txnDataName(), command)));
-//                    return;
-//                }
-
-            results.add(AsyncChains.ofCallable(Stage.ACCORD_MIGRATION.executor(), () -> {
-                TxnData result = new TxnData();
-                try (PartitionIterator iterator = StorageProxy.getRangeSlice(command, consistencyLevel, this, requestTime))
-                {
-                    TxnDataRangeValue value = new TxnDataRangeValue();
-                    while (iterator.hasNext())
-                    {
-                        try (RowIterator partition = iterator.next())
-                        {
-                            FilteredPartition filtered = FilteredPartition.create(partition);
-                            if (filtered.hasRows() || command.selectsFullPartition())
-                                value.add(filtered);
-                        }
-                    }
-                    result.put(TxnData.txnDataName(TxnDataNameKind.USER), value);
-                }
-                return result;
-            }));
         });
         return results;
     }
 
     private AsyncChain<Data> readChains()
     {
-        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc());
         // TODO (expected): use normal query nano time
         Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
 
-        List<AsyncChain<Data>> results;
-        if (txn.keys().domain().isKey())
-            results = keyReadChains(nowInSeconds, requestTime);
-        else
-            results = rangeReadChains(nowInSeconds, requestTime);
-
+        List<AsyncChain<Data>> results = readChains(requestTime);
         if (results.isEmpty())
             return AsyncChains.success(new TxnData());
 
@@ -431,9 +427,20 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
     }
 
     @Override
-    public ReadCommand maybeAllowOutOfRangeReads(ReadCommand readCommand)
+    public ReadCommand maybeAllowOutOfRangeReads(ReadCommand readCommand, ConsistencyLevel cl)
     {
-        return readCommand.allowOutOfRangeReads();
+        // Reading from a single coordinator so there is no reconciliation at the coordinator and filtering/limits
+        // need to be pushed down to query execution
+        boolean withoutReconciliation = cl == null || cl == ConsistencyLevel.ONE;
+        // Really just want to enable allowPotentialTxnConflicts without changing anything else
+        // but didn't want to add another method for constructing a modified read command
+        if (readCommand instanceof SinglePartitionReadCommand)
+            return ((SinglePartitionReadCommand)readCommand).withTransactionalSettings(withoutReconciliation, readCommand.nowInSec());
+        else
+        {
+            PartitionRangeReadCommand rangeCommand = ((PartitionRangeReadCommand)readCommand);
+            return rangeCommand.withTransactionalSettings(readCommand.nowInSec(), rangeCommand.dataRange().keyRange(), true, withoutReconciliation);
+        }
     }
 
     // Provide request callbacks with a way to send maximal commits on Insufficient responses

src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java

@@ -19,22 +19,34 @@
 package org.apache.cassandra.service.accord.interop;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
 import accord.api.Data;
 import accord.local.Node;
 import accord.local.SafeCommandStore;
-import accord.messages.ReadData;
 import accord.messages.MessageType;
+import accord.messages.ReadData;
 import accord.primitives.PartialTxn;
 import accord.primitives.Participants;
+import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.primitives.Routables.Slice;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.topology.Topologies;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadCommandVerbHandler;
 import org.apache.cassandra.db.ReadResponse;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
@@ -46,17 +58,26 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializers.ReadDataSerializer;
+import org.apache.cassandra.service.accord.txn.TxnNamedRead;
+import org.apache.cassandra.service.accord.txn.TxnRead;
+import org.apache.cassandra.utils.Pair;
 
 import static accord.primitives.SaveStatus.PreApplied;
 import static accord.primitives.SaveStatus.ReadyToExecute;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
 
 public class AccordInteropRead extends ReadData
 {
-    public static final IVersionedSerializer<AccordInteropRead> requestSerializer = new ReadDataSerializer<>()
+
+    public static final IVersionedSerializer<AccordInteropRead> requestSerializer = new ReadDataSerializer<AccordInteropRead>()
     {
         @Override
         public void serialize(AccordInteropRead read, DataOutputPlus out, int version) throws IOException
@@ -64,7 +85,7 @@ public class AccordInteropRead extends ReadData
             CommandSerializers.txnId.serialize(read.txnId, out, version);
             KeySerializers.participants.serialize(read.scope, out, version);
             out.writeUnsignedVInt(read.executeAtEpoch);
-            SinglePartitionReadCommand.serializer.serialize(read.command, out, version);
+            ReadCommand.serializer.serialize(read.command, out, version);
         }
 
         @Override
@@ -73,7 +94,7 @@ public class AccordInteropRead extends ReadData
             TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
             Participants<?> scope = KeySerializers.participants.deserialize(in, version);
             long executeAtEpoch = in.readUnsignedVInt();
-            SinglePartitionReadCommand command = (SinglePartitionReadCommand) SinglePartitionReadCommand.serializer.deserialize(in, version);
+            ReadCommand command = ReadCommand.serializer.deserialize(in, version);
             return new AccordInteropRead(txnId, scope, executeAtEpoch, command);
         }
 
@@ -83,7 +104,7 @@ public class AccordInteropRead extends ReadData
             return CommandSerializers.txnId.serializedSize(read.txnId, version)
                    + KeySerializers.participants.serializedSize(read.scope, version)
                    + TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
-                   + SinglePartitionReadCommand.serializer.serializedSize(read.command, version);
+                   + ReadCommand.serializer.serializedSize(read.command, version);
         }
     };
 
@@ -91,12 +112,15 @@ public class AccordInteropRead extends ReadData
 
     private static class LocalReadData implements Data
     {
+        private static final Comparator<Pair<AccordRoutingKey, ReadResponse>> RESPONSE_COMPARATOR = Comparator.comparing(Pair::left);
+
         static final IVersionedSerializer<LocalReadData> serializer = new IVersionedSerializer<>()
         {
             @Override
             public void serialize(LocalReadData data, DataOutputPlus out, int version) throws IOException
             {
-                ReadResponse.serializer.serialize(data.response, out, version);
+                data.ensureRemoteResponse();
+                ReadResponse.serializer.serialize(data.remoteResponse, out, version);
             }
 
             @Override
@@ -108,27 +132,76 @@ public class AccordInteropRead extends ReadData
             @Override
             public long serializedSize(LocalReadData data, int version)
             {
-                return ReadResponse.serializer.serializedSize(data.response, version);
+                data.ensureRemoteResponse();
+                return ReadResponse.serializer.serializedSize(data.remoteResponse, version);
             }
         };
 
-        final ReadResponse response;
+        // Will be null at coordinator
+        List<Pair<AccordRoutingKey, ReadResponse>> localResponses;
+        // Will be null at coordinator
+        final ReadCommand readCommand;
+        // Will be not null at coordinator, but null at the node creating the response until it serialized
+        ReadResponse remoteResponse;
 
-        public LocalReadData(ReadResponse response)
+        public LocalReadData(@Nullable AccordRoutingKey start, @Nonnull ReadResponse response, @Nonnull ReadCommand readCommand)
         {
-            this.response = response;
+            checkNotNull(response, "response is null");
+            checkNotNull(readCommand, "readCommand is null");
+            localResponses = ImmutableList.of(Pair.create(start, response));
+            this.readCommand = readCommand;
+            this.remoteResponse = null;
+        }
+
+        public LocalReadData(@Nonnull ReadResponse remoteResponse)
+        {
+            checkNotNull(remoteResponse);
+            this.remoteResponse = remoteResponse;
+            readCommand = null;
         }
 
         @Override
         public String toString()
         {
-            return "LocalReadData{" + response + '}';
+            if (localResponses != null)
+               return "LocalReadData{" + localResponses + '}';
+            else
+                return "LocalReadData{" + remoteResponse + '}';
         }
 
         @Override
         public Data merge(Data data)
         {
-            throw new IllegalStateException("Should only ever be a single partition");
+            checkState(remoteResponse == null, "Already serialized");
+            checkState(readCommand.isRangeRequest(), "Should only ever be a single partition");
+            LocalReadData other = (LocalReadData)data;
+            checkState(readCommand == other.readCommand, "Should share the same ReadCommand");
+            if (localResponses.size() == 1)
+            {
+                List<Pair<AccordRoutingKey, ReadResponse>> merged = new ArrayList<>();
+                merged.add(localResponses.get(0));
+                localResponses = merged;
+            }
+            localResponses.addAll(other.localResponses);
+            return this;
+        }
+
+        private void ensureRemoteResponse()
+        {
+            if (remoteResponse != null)
+                return;
+            // Range reads will be spread across command stores and need to be merged in token order
+            List<Pair<AccordRoutingKey, ReadResponse>> responses = localResponses;
+            if (responses.size() == 1)
+            {
+                remoteResponse = responses.get(0).right;
+            }
+            else
+            {
+                responses = new ArrayList(responses);
+                Collections.sort(responses, RESPONSE_COMPARATOR);
+                remoteResponse = ReadResponse.merge(Lists.transform(responses, Pair::right), readCommand);
+            }
         }
     }
 
@@ -142,21 +215,21 @@ public class AccordInteropRead extends ReadData
         @Override
         ReadResponse convertResponse(ReadOk ok)
         {
-            return ((LocalReadData) ok.data).response;
+            return ((LocalReadData)ok.data).remoteResponse;
         }
     }
 
     private static final ExecuteOn EXECUTE_ON = new ExecuteOn(ReadyToExecute, PreApplied);
 
-    final SinglePartitionReadCommand command;
+    protected final ReadCommand command;
 
-    public AccordInteropRead(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> scope, long executeAtEpoch, SinglePartitionReadCommand command)
+    public AccordInteropRead(Node.Id to, Topologies topologies, TxnId txnId, Participants<?> scope, long executeAtEpoch, ReadCommand command)
     {
         super(to, topologies, txnId, scope, executeAtEpoch);
         this.command = command;
     }
 
-    public AccordInteropRead(TxnId txnId, Participants<?> scope, long executeAtEpoch, SinglePartitionReadCommand command)
+    public AccordInteropRead(TxnId txnId, Participants<?> scope, long executeAtEpoch, ReadCommand command)
     {
         super(txnId, scope, executeAtEpoch);
         this.command = command;
@@ -171,8 +244,42 @@ public class AccordInteropRead extends ReadData
     @Override
     protected AsyncChain<Data> beginRead(SafeCommandStore safeStore, Timestamp executeAt, PartialTxn txn, Ranges unavailable)
     {
-        // TODO (required): subtract unavailable ranges, either from read or from response (or on coordinator)
-        return AsyncChains.ofCallable(Stage.READ.executor(), () -> new LocalReadData(ReadCommandVerbHandler.instance.doRead(command, false)));
+        TxnRead txnRead = (TxnRead)txn.read();
+        Ranges ranges = safeStore.ranges().allAt(executeAt).without(unavailable).intersecting(scope, Slice.Minimal);
+        long nowInSeconds = TxnNamedRead.nowInSeconds(executeAt);
+        List<AsyncChain<Data>> chains = new ArrayList<>(ranges.size());
+        for (Range r : ranges)
+        {
+            ReadCommand readCommand = this.command;
+            AccordRoutingKey routingKey = null;
+            final ReadCommand readCommandFinal;
+            if (readCommand.isRangeRequest())
+            {
+                // This path can have a subrange we have never seen before provided by short read protection or read repair so we need to
+                // calculate the intersection with this instance of the command store and the actual command if it is not empty we
+                // will need to execute it
+                TokenRange commandRange = TxnNamedRead.boundsAsAccordRange(readCommand.dataRange().keyRange(), readCommand.metadata().id);
+                Range intersection = commandRange.intersection(r);
+                if (intersection == null)
+                    continue;
+                readCommandFinal = TxnNamedRead.commandForSubrange((PartitionRangeReadCommand) readCommand, intersection, txnRead.cassandraConsistencyLevel(), nowInSeconds);
+                routingKey = ((TokenRange)r).start();
+            }
+            else
+            {
+                SinglePartitionReadCommand singlePartitionReadCommand = ((SinglePartitionReadCommand)readCommand);
+                if (!r.contains(new TokenKey(singlePartitionReadCommand.metadata().id, singlePartitionReadCommand.partitionKey().getToken())))
+                    continue;
+                readCommandFinal = ((SinglePartitionReadCommand)readCommand).withTransactionalSettings(TxnNamedRead.readsWithoutReconciliation(txnRead.cassandraConsistencyLevel()), nowInSeconds);
+            }
+            AccordRoutingKey routingKeyFinal = routingKey;
+            chains.add(AsyncChains.ofCallable(Stage.READ.executor(), () -> new LocalReadData(routingKeyFinal, ReadCommandVerbHandler.instance.doRead(readCommandFinal, false), readCommand)));
+        }
+
+        if (chains.isEmpty())
+            return AsyncChains.success(null);
+
+        return AsyncChains.reduce(chains, Data::merge);
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java

@@ -58,6 +58,7 @@ public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply
 
     abstract T convertResponse(ReadOk ok);
 
+    @Override
     public void onSuccess(Node.Id from, ReadReply reply)
     {
         Invariants.requireArgument(from.equals(id));
@@ -78,6 +79,7 @@ public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply
         }
     }
 
+    @Override
     public void onFailure(Node.Id from, Throwable failure)
     {
         RequestFailure requestFailure;
@@ -90,6 +92,7 @@ public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply
         wrapped.onFailure(endpoint, requestFailure);
     }
 
+    @Override
     public boolean onCallbackFailure(Node.Id from, Throwable failure)
     {
         wrapped.onFailure(endpoint, RequestFailure.UNKNOWN);

src/java/org/apache/cassandra/service/accord/interop/AccordInteropStableThenRead.java

@@ -39,7 +39,7 @@ import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.topology.Topologies;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -78,7 +78,7 @@ public class AccordInteropStableThenRead extends AccordInteropRead
                 DepsSerializers.partialDeps.serialize(read.partialDeps, out, version);
             if (read.kind.withTxn == HasTxn)
                 KeySerializers.fullRoute.serialize(read.route, out, version);
-            SinglePartitionReadCommand.serializer.serialize(read.command, out, version);
+            ReadCommand.serializer.serialize(read.command, out, version);
         }
 
         @Override
@@ -92,7 +92,7 @@ public class AccordInteropStableThenRead extends AccordInteropRead
             PartialTxn partialTxn = kind.withTxn == NoTxn ? null : CommandSerializers.nullablePartialTxn.deserialize(in, version);
             PartialDeps partialDeps = kind.withDeps == NoDeps ? null : DepsSerializers.partialDeps.deserialize(in, version);
             FullRoute < ?> route = kind.withTxn == HasTxn ? KeySerializers.fullRoute.deserialize(in, version) : null;
-            SinglePartitionReadCommand command = (SinglePartitionReadCommand) SinglePartitionReadCommand.serializer.deserialize(in, version);
+            ReadCommand command = ReadCommand.serializer.deserialize(in, version);
             return new AccordInteropStableThenRead(txnId, scope, kind, minEpoch, executeAt, partialTxn, partialDeps, route, command);
         }
 
@@ -107,7 +107,7 @@ public class AccordInteropStableThenRead extends AccordInteropRead
                    + (read.kind.withTxn == NoTxn ? 0 : CommandSerializers.nullablePartialTxn.serializedSize(read.partialTxn, version))
                    + (read.kind.withDeps != HasDeps ? 0 : DepsSerializers.partialDeps.serializedSize(read.partialDeps, version))
                    + (read.kind.withTxn != HasTxn ? 0 : KeySerializers.fullRoute.serializedSize(read.route, version))
-                   + SinglePartitionReadCommand.serializer.serializedSize(read.command, version);
+                   + ReadCommand.serializer.serializedSize(read.command, version);
         }
     };
 
@@ -121,18 +121,18 @@ public class AccordInteropStableThenRead extends AccordInteropRead
     public final @Nullable PartialDeps partialDeps;
     public final @Nullable FullRoute<?> route;
 
-    public AccordInteropStableThenRead(Node.Id to, Topologies topologies, TxnId txnId, Commit.Kind kind, Timestamp executeAt, Txn txn, Deps deps, FullRoute<?> route, SinglePartitionReadCommand command)
+    public AccordInteropStableThenRead(Node.Id to, Topologies topologies, TxnId txnId, Commit.Kind kind, Timestamp executeAt, Txn txn, Deps deps, FullRoute<?> route, ReadCommand command)
     {
         super(to, topologies, txnId, route, executeAt.epoch(), command);
         this.kind = kind;
         this.minEpoch = topologies.oldestEpoch();
         this.executeAt = executeAt;
-        this.partialTxn = kind.withTxn.select(txn, scope, topologies, txnId, to);
-        this.partialDeps = kind.withDeps.select(deps, scope);
+        this.partialTxn = kind.withTxn.select(txn, route, topologies, txnId, to);
+        this.partialDeps = kind.withDeps.select(deps, route);
         this.route = kind.withTxn.select(route);
     }
 
-    public AccordInteropStableThenRead(TxnId txnId, Participants<?> scope, Commit.Kind kind, long minEpoch, Timestamp executeAt, @Nullable PartialTxn partialTxn, @Nullable PartialDeps partialDeps, @Nullable FullRoute<?> route, SinglePartitionReadCommand command)
+    public AccordInteropStableThenRead(TxnId txnId, Participants<?> scope, Commit.Kind kind, long minEpoch, Timestamp executeAt, @Nullable PartialTxn partialTxn, @Nullable PartialDeps partialDeps, @Nullable FullRoute<?> route, ReadCommand command)
     {
         super(txnId, scope, executeAt.epoch(), command);
         this.minEpoch = minEpoch;

src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java

@@ -45,6 +45,7 @@ import accord.primitives.Ranges;
 import accord.primitives.RoutableKey;
 import accord.primitives.Route;
 import accord.primitives.RoutingKeys;
+import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Unseekables;
 import accord.primitives.Unseekables.UnseekablesKind;
@@ -418,6 +419,51 @@ public class KeySerializers
         }
     }
 
+    public static final IVersionedSerializer<Seekable> seekable = new IVersionedSerializer<>()
+    {
+        @Override
+        public void serialize(Seekable seekable, DataOutputPlus out, int version) throws IOException
+        {
+            switch (seekable.domain())
+            {
+                default: throw new AssertionError();
+                case Key:
+                    out.writeByte(0);
+                    PartitionKey.serializer.serialize((PartitionKey) seekable, out, version);
+                    break;
+                case Range:
+                    out.writeByte(1);
+                    TokenRange.serializer.serialize((TokenRange) seekable, out, version);
+                    break;
+            }
+        }
+
+        @Override
+        public Seekable deserialize(DataInputPlus in, int version) throws IOException
+        {
+            byte b = in.readByte();
+            switch (b)
+            {
+                default: throw new IOException("Corrupted input: expected byte 1 or 2, received " + b);
+                case 0: return PartitionKey.serializer.deserialize(in, version);
+                case 1: return TokenRange.serializer.deserialize(in, version);
+            }
+        }
+
+        @Override
+        public long serializedSize(Seekable seekable, int version)
+        {
+            switch (seekable.domain())
+            {
+                default: throw new AssertionError();
+                case Key:
+                    return 1 + PartitionKey.serializer.serializedSize((PartitionKey) seekable, version);
+                case Range:
+                    return 1 + TokenRange.serializer.serializedSize((TokenRange) seekable, version);
+            }
+        }
+    };
+
     public static class AbstractSeekablesSerializer implements IVersionedSerializer<Seekables<?, ?>>
     {
         final IVersionedSerializer<Keys> keys;

src/java/org/apache/cassandra/service/accord/txn/AbstractKeySorted.java

@@ -28,7 +28,14 @@ import java.util.stream.Collectors;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterators;
 
+import accord.api.Key;
 import accord.primitives.Keys;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
+import accord.primitives.Seekable;
+import accord.primitives.Seekables;
+import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 
 /**
@@ -38,30 +45,55 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
 {
     public static final String ITEMS_OUT_OF_ORDER_MESSAGE = "Items are out of order ([%s] %s >= [%s] %s)";
 
-    protected final Keys itemKeys;
+    protected final Seekables itemKeys;
     protected final T[] items;
 
-    public AbstractKeySorted(T[] items)
+    public AbstractKeySorted(T[] items, Domain domain)
     {
         this.items = items;
-        this.itemKeys = extractItemKeys();
+        this.itemKeys = extractItemKeys(domain);
     }
 
-    public AbstractKeySorted(List<T> items)
+    public AbstractKeySorted(List<T> items, Domain domain)
     {
         T[] arr = newArray(items.size());
         items.toArray(arr);
-        Arrays.sort(arr, this::compare);
         this.items = arr;
-        this.itemKeys = extractItemKeys();
+        switch (domain)
+        {
+            case Key:
+                Arrays.sort(arr, this::compareKey);
+                break;
+            case Range:
+                Arrays.sort(arr, this::compareRange);
+                break;
+            default:
+                throw new IllegalStateException("Unhandled domain " + domain);
+        }
+        this.itemKeys = extractItemKeys(domain);
     }
 
-    private Keys extractItemKeys()
+    private Seekables extractItemKeys(Domain domain)
     {
-        PartitionKey[] keys = new PartitionKey[items.length];
-        for (int i = 0 ; i < keys.length ; ++i)
-            keys[i] = getKey(items[i]);
-        return Keys.ofSorted(keys);
+        switch (domain)
+        {
+            case Key:
+                if (items.length == 0)
+                    return Keys.EMPTY;
+                PartitionKey[] keys = new PartitionKey[items.length];
+                for (int i = 0 ; i < keys.length; i++)
+                    keys[i] = (PartitionKey)getKey(items[i]);
+                return Keys.ofSorted(keys);
+            case Range:
+                if (items.length == 0)
+                    return Ranges.EMPTY;
+                TokenRange[] ranges = new TokenRange[items.length];
+                for (int i = 0 ; i < ranges.length; i++)
+                    ranges[i] = (TokenRange)getKey(items[i]);
+                return Ranges.ofSortedAndDeoverlapped(ranges);
+            default:
+                throw new IllegalStateException("Unhandled domain " + domain);
+        }
     }
 
     @Override
@@ -94,7 +126,7 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
     }
 
     @VisibleForTesting
-    public Keys keys()
+    public Seekables keys()
     {
         return itemKeys;
     }
@@ -104,25 +136,51 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
      */
     abstract int compareNonKeyFields(T left, T right);
 
-    abstract PartitionKey getKey(T item);
+    abstract Seekable getKey(T item);
     abstract T[] newArray(int size);
 
-    public int compare(T left, T right)
+    abstract Domain domain();
+
+    public int compareKey(T left, T right)
     {
-        int cmp = getKey(left).compareTo(getKey(right));
+        int cmp = ((PartitionKey)getKey(left)).compareTo(((PartitionKey)getKey(right)));
+        return cmp != 0 ? cmp : compareNonKeyFields(left, right);
+    }
+
+    public int compareRange(T left, T right)
+    {
+        int cmp = ((TokenRange)getKey(left)).compareTo(((TokenRange)getKey(right)));
         return cmp != 0 ? cmp : compareNonKeyFields(left, right);
     }
 
     @VisibleForTesting
     void validateOrder()
     {
-        for (int i = 1; i < items.length; i++)
+        Domain domain = getKey(items[0]).domain();
+        switch (domain)
         {
-            T prev = items[i-1];
-            T next = items[i];
+            case Key:
+                for (int i = 1; i < items.length; i++)
+                {
+                    T prev = items[i-1];
+                    T next = items[i];
 
-            if (compare(prev, next) >= 0)
-                throw new IllegalStateException(String.format(ITEMS_OUT_OF_ORDER_MESSAGE, i - 1, prev, i, next));
+                    if (compareKey(prev, next) >= 0)
+                        throw new IllegalStateException(String.format(ITEMS_OUT_OF_ORDER_MESSAGE, i - 1, prev, i, next));
+                }
+                break;
+            case Range:
+                for (int i = 1; i < items.length; i++)
+                {
+                    T prev = items[i-1];
+                    T next = items[i];
+
+                    if (compareRange(prev, next) >= 0)
+                        throw new IllegalStateException(String.format(ITEMS_OUT_OF_ORDER_MESSAGE, i - 1, prev, i, next));
+                }
+                break;
+            default:
+                throw new IllegalStateException("Unhandled domain " + domain);
         }
     }
 
@@ -131,16 +189,55 @@ public abstract class AbstractKeySorted<T> implements Iterable<T>
         return items.length;
     }
 
-    public void forEachWithKey(PartitionKey key, Consumer<T> consumer)
+    public void forEachWithKey(Seekable key, Consumer<T> consumer)
     {
-        for (int i = firstPossibleKeyIdx(key); i < items.length && getKey(items[i]).equals(key); i++)
-            consumer.accept(items[i]);
+        switch (key.domain())
+        {
+            case Key:
+                for (int i = firstPossibleKeyIdx((PartitionKey) key); i < items.length; i++)
+                {
+                    Key itemKey = (Key)getKey(items[i]);
+                    if (key.equals(itemKey))
+                        consumer.accept(items[i]);
+                    else
+                        break;
+                }
+                break;
+            case Range:
+                TokenRange range = (TokenRange) key;
+                for (int i = firstPossibleRangeIdx(range); i < items.length; i++)
+                {
+                    Range itemRange = (Range) getKey(items[i]);
+                    if (range.compareIntersecting(itemRange) == 0)
+                        consumer.accept(items[i]);
+                    else
+                        break;
+                }
+                break;
+            default:
+                throw new IllegalStateException("Unhandled domain " + key.domain());
+        }
+    }
+
+    private int firstPossibleRangeIdx(TokenRange range)
+    {
+        int idx = Arrays.binarySearch(items, range, (l, r) -> {
+            Range itemRange = (Range)getKey((T) l);
+            if (itemRange.compareIntersecting((TokenRange)r) == 0)
+                return 1;
+            if (((TokenRange) r).end().compareTo(itemRange.end()) > 0)
+                return 1;
+            else
+                return -1;
+        });
+
+        return -1 - idx;
     }
 
     private int firstPossibleKeyIdx(PartitionKey key)
     {
         int idx = Arrays.binarySearch(items, key, (l, r) -> {
-            PartitionKey lk = getKey((T) l);
+            PartitionKey lk = (PartitionKey) getKey((T) l);
             PartitionKey rk = (PartitionKey) r;
             int cmp = lk.compareTo(rk);
             return cmp != 0 ? cmp : 1;

src/java/org/apache/cassandra/service/accord/txn/RetryWithNewProtocolResult.java

@@ -18,22 +18,16 @@
 
 package org.apache.cassandra.service.accord.txn;
 
-import org.apache.cassandra.tcm.Epoch;
-import org.apache.cassandra.utils.ObjectSizes;
-
 /**
  * Potentially returned by any transaction that tries to execute in an Epoch
  * where the range has migrated away from Accord
  */
 public class RetryWithNewProtocolResult implements TxnResult
 {
-    private static final long EMPTY_SIZE = ObjectSizes.measure(new RetryWithNewProtocolResult(null));
+    public static final RetryWithNewProtocolResult instance = new RetryWithNewProtocolResult();
 
-    public final Epoch epoch;
-
-    RetryWithNewProtocolResult(Epoch epoch)
+    private RetryWithNewProtocolResult()
     {
-        this.epoch = epoch;
     }
 
     @Override
@@ -45,6 +39,6 @@ public class RetryWithNewProtocolResult implements TxnResult
     @Override
     public long estimatedSizeOnHeap()
     {
-        return EMPTY_SIZE + epoch.estimatedSizeOnHeap();
+        return 0;
     }
 }

src/java/org/apache/cassandra/service/accord/txn/TxnKeyRead.java

@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord.txn;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import com.google.common.collect.ImmutableList;
-
-import accord.api.Data;
-import accord.api.DataStore;
-import accord.api.Read;
-import accord.local.SafeCommandStore;
-import accord.primitives.Keys;
-import accord.primitives.Participants;
-import accord.primitives.Ranges;
-import accord.primitives.Seekable;
-import accord.primitives.Timestamp;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.SinglePartitionReadCommand;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.utils.ObjectSizes;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer;
-import static org.apache.cassandra.service.accord.IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS;
-import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.CAS_READ;
-import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.USER;
-import static org.apache.cassandra.service.accord.txn.TxnData.txnDataName;
-import static org.apache.cassandra.utils.ArraySerializers.deserializeArray;
-import static org.apache.cassandra.utils.ArraySerializers.serializeArray;
-import static org.apache.cassandra.utils.ArraySerializers.serializedArraySize;
-import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable;
-import static org.apache.cassandra.utils.NullableSerializer.serializeNullable;
-import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize;
-
-public class TxnKeyRead extends AbstractKeySorted<TxnNamedRead> implements TxnRead
-{
-    public static final TxnKeyRead EMPTY = new TxnKeyRead(new TxnNamedRead[0], null);
-    private static final long EMPTY_SIZE = ObjectSizes.measure(EMPTY);
-    private static final Comparator<TxnNamedRead> TXN_NAMED_READ_KEY_COMPARATOR = Comparator.comparing(TxnNamedRead::key);
-
-    // Cassandra's consistency level used by Accord to safely read data written outside of Accord
-    @Nullable
-    private final ConsistencyLevel cassandraConsistencyLevel;
-
-    private TxnKeyRead(@Nonnull TxnNamedRead[] items, @Nullable ConsistencyLevel cassandraConsistencyLevel)
-    {
-        super(items);
-        checkNotNull(items, "items is null");
-        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read");
-        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
-    }
-
-    private TxnKeyRead(@Nonnull List<TxnNamedRead> items, @Nullable ConsistencyLevel cassandraConsistencyLevel)
-    {
-        super(items);
-        checkNotNull(items, "items is null");
-        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read");
-        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
-    }
-
-    public static TxnKeyRead createTxnRead(@Nonnull List<TxnNamedRead> items, @Nullable ConsistencyLevel consistencyLevel)
-    {
-        items.sort(Comparator.comparing(TxnNamedRead::key));
-        return new TxnKeyRead(items, consistencyLevel);
-    }
-
-    public static TxnKeyRead createSerialRead(List<SinglePartitionReadCommand> readCommands, ConsistencyLevel consistencyLevel)
-    {
-        List<TxnNamedRead> reads = new ArrayList<>(readCommands.size());
-        for (int i = 0; i < readCommands.size(); i++)
-            reads.add(new TxnNamedRead(txnDataName(USER, i), readCommands.get(i)));
-        reads.sort(TXN_NAMED_READ_KEY_COMPARATOR);
-        return new TxnKeyRead(reads, consistencyLevel);
-    }
-
-    public static TxnKeyRead createCasRead(SinglePartitionReadCommand readCommand, ConsistencyLevel consistencyLevel)
-    {
-        TxnNamedRead read = new TxnNamedRead(txnDataName(CAS_READ), readCommand);
-        return new TxnKeyRead(ImmutableList.of(read), consistencyLevel);
-    }
-
-    // A read that declares it will read from keys but doesn't actually read any data so dependent transactions will
-    // still be applied first
-    public static TxnKeyRead createNoOpRead(Keys keys)
-    {
-        List<TxnNamedRead> reads = new ArrayList<>(keys.size());
-        for (int i = 0; i < keys.size(); i++)
-            reads.add(new TxnNamedRead(txnDataName(USER, i), (PartitionKey)keys.get(i), null));
-        return new TxnKeyRead(reads, null);
-    }
-
-    @Override
-    public long estimatedSizeOnHeap()
-    {
-        long size = EMPTY_SIZE;
-        for (TxnNamedRead read : items)
-            size += read.estimatedSizeOnHeap();
-        return size;
-    }
-
-    @Override
-    int compareNonKeyFields(TxnNamedRead left, TxnNamedRead right)
-    {
-        return Integer.compare(left.txnDataName(), right.txnDataName());
-    }
-
-    @Override
-    PartitionKey getKey(TxnNamedRead read)
-    {
-        return read.key();
-    }
-
-    @Override
-    TxnNamedRead[] newArray(int size)
-    {
-        return new TxnNamedRead[size];
-    }
-
-    @Override
-    public Keys keys()
-    {
-        return itemKeys;
-    }
-
-    @Override
-    public ConsistencyLevel cassandraConsistencyLevel()
-    {
-        return cassandraConsistencyLevel;
-    }
-
-    @Override
-    public Read slice(Ranges ranges)
-    {
-        return intersecting(itemKeys.slice(ranges));
-    }
-
-    @Override
-    public Read intersecting(Participants<?> participants)
-    {
-        return intersecting(itemKeys.intersecting(participants));
-    }
-
-    private Read intersecting(Keys select)
-    {
-        Keys keys = itemKeys.intersecting(select);
-        List<TxnNamedRead> reads = new ArrayList<>(keys.size());
-
-        for (TxnNamedRead read : items)
-            if (keys.contains(read.key()))
-                reads.add(read);
-
-        return createTxnRead(reads, cassandraConsistencyLevel);
-    }
-
-    @Override
-    public Read merge(Read read)
-    {
-        List<TxnNamedRead> reads = new ArrayList<>(items.length);
-        Collections.addAll(reads, items);
-
-        for (TxnNamedRead namedRead : (TxnKeyRead) read)
-            if (!reads.contains(namedRead))
-                reads.add(namedRead);
-
-        return createTxnRead(reads, cassandraConsistencyLevel);
-    }
-
-    @Override
-    public void unmemoize()
-    {
-        for (TxnNamedRead read : items)
-            read.unmemoize();
-    }
-
-    @Override
-    public AsyncChain<Data> read(Seekable key, SafeCommandStore safeStore, Timestamp executeAt, DataStore store)
-    {
-        // Set to null since we don't need it and interop can pass in null
-        safeStore = null;
-
-        List<AsyncChain<Data>> results = new ArrayList<>();
-        forEachWithKey((PartitionKey) key, read -> results.add(read.read(cassandraConsistencyLevel, executeAt)));
-
-        if (results.isEmpty())
-            // Result type must match everywhere
-            return AsyncChains.success(new TxnData());
-
-        if (results.size() == 1)
-            return results.get(0);
-
-        return AsyncChains.reduce(results, Data::merge);
-    }
-
-    @Override
-    public Kind kind()
-    {
-        return Kind.key;
-    }
-
-    public static final TxnReadSerializer<TxnKeyRead> serializer = new TxnReadSerializer<TxnKeyRead>()
-    {
-        @Override
-        public void serialize(TxnKeyRead read, DataOutputPlus out, int version) throws IOException
-        {
-            serializeArray(read.items, out, version, TxnNamedRead.serializer);
-            serializeNullable(read.cassandraConsistencyLevel, out, version, consistencyLevelSerializer);
-        }
-
-        @Override
-        public TxnKeyRead deserialize(DataInputPlus in, int version) throws IOException
-        {
-            TxnNamedRead[] items = deserializeArray(in, version, TxnNamedRead.serializer, TxnNamedRead[]::new);
-            ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer);
-            return new TxnKeyRead(items, consistencyLevel);
-        }
-
-        @Override
-        public long serializedSize(TxnKeyRead read, int version)
-        {
-            long size = 0;
-            size += serializedArraySize(read.items, version, TxnNamedRead.serializer);
-            size += serializedNullableSize(read.cassandraConsistencyLevel, version, consistencyLevelSerializer);
-            return size;
-        }
-    };
-}

src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java

@@ -29,6 +29,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.api.Data;
+import accord.primitives.Range;
+import accord.primitives.Seekable;
 import accord.primitives.Timestamp;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
@@ -36,20 +38,38 @@ import accord.utils.async.AsyncResults;
 import org.apache.cassandra.concurrent.DebuggableTask;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.SinglePartitionReadCommand;
 import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.db.partitions.FilteredPartition;
 import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.dht.Token.KeyBound;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordObjectSizes;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.service.accord.serializers.KeySerializers;
+import org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.MonotonicClock;
 import org.apache.cassandra.utils.ObjectSizes;
 
+import static com.google.common.base.Preconditions.checkState;
 import static org.apache.cassandra.utils.ByteBufferUtil.readWithVIntLength;
 import static org.apache.cassandra.utils.ByteBufferUtil.serializedSizeWithVIntLength;
 import static org.apache.cassandra.utils.ByteBufferUtil.writeWithVIntLength;
@@ -59,10 +79,10 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
     @SuppressWarnings("unused")
     private static final Logger logger = LoggerFactory.getLogger(TxnNamedRead.class);
 
-    private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnNamedRead(0, null, null));
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new TxnNamedRead(0, null, (ByteBuffer) null));
 
     private final int name;
-    private final PartitionKey key;
+    private final Seekable key;
 
     public TxnNamedRead(int name, @Nullable SinglePartitionReadCommand value)
     {
@@ -71,7 +91,51 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
         this.key = new PartitionKey(value.metadata().id, value.partitionKey());
     }
 
-    public TxnNamedRead(int name, PartitionKey key, ByteBuffer bytes)
+    public static TokenRange boundsAsAccordRange(AbstractBounds<PartitionPosition> range, TableId tableId)
+    {
+        // Should already have been unwrapped
+        checkState(!AbstractBounds.strictlyWrapsAround(range.left, range.right));
+
+        // Read commands can contain a mix of different kinds of bounds to facilitate paging
+        // and we need to communicate that to Accord as its own ranges. This uses
+        // TokenKey, SentinelKey, and MinTokenKey and sticks exclusively with left exclusive/right inclusive
+        // ranges rather add more types of ranges to the mix
+        // MinTokenKey allows emulating inclusive left and exclusive right with Range
+        boolean inclusiveLeft = range.inclusiveLeft();
+        PartitionPosition startPP = range.left;
+        boolean startIsMinKeyBound = startPP.getClass() == KeyBound.class ? ((KeyBound)startPP).isMinimumBound : false;
+        Token startToken = startPP.getToken();
+        AccordRoutingKey startAccordRoutingKey;
+        if (startToken.isMinimum() && inclusiveLeft)
+            startAccordRoutingKey = SentinelKey.min(tableId);
+        else if (inclusiveLeft || startIsMinKeyBound)
+            startAccordRoutingKey = new MinTokenKey(tableId, startToken);
+        else
+            startAccordRoutingKey = new TokenKey(tableId, startToken);
+
+        boolean inclusiveRight = range.inclusiveRight();
+        PartitionPosition endPP = range.right;
+        boolean endIsMinKeyBound = endPP.getClass() == KeyBound.class ? ((KeyBound)endPP).isMinimumBound : false;
+        Token stopToken = range.right.getToken();
+        AccordRoutingKey stopAccordRoutingKey;
+        if (stopToken.isMinimum())
+            stopAccordRoutingKey = SentinelKey.max(tableId);
+        else if (inclusiveRight && !endIsMinKeyBound)
+            stopAccordRoutingKey = new TokenKey(tableId, stopToken);
+        else
+            stopAccordRoutingKey = new MinTokenKey(tableId, stopToken);
+        return TokenRange.create(startAccordRoutingKey, stopAccordRoutingKey);
+    }
+
+    public TxnNamedRead(int name, AbstractBounds<PartitionPosition> range, PartitionRangeReadCommand value)
+    {
+        super(value);
+        TableId tableId = value.metadata().id;
+        this.name = name;
+        this.key = boundsAsAccordRange(range, tableId);
+    }
+
+    public TxnNamedRead(int name, Seekable key, ByteBuffer bytes)
     {
         super(bytes);
         this.name = name;
@@ -80,13 +144,16 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
 
     public long estimatedSizeOnHeap()
     {
-        return EMPTY_SIZE + key.estimatedSizeOnHeap() + (bytes() != null ? ByteBufferUtil.estimatedSizeOnHeap(bytes()) : 0);
+        long size = EMPTY_SIZE;
+        size += AccordObjectSizes.seekable(key);
+        size += (bytes() != null ? ByteBufferUtil.estimatedSizeOnHeap(bytes()) : 0);
+        return size;
     }
 
     @Override
     protected IVersionedSerializer<ReadCommand> serializer()
     {
-        return SinglePartitionReadCommand.serializer;
+        return ReadCommand.serializer;
     }
 
     @Override
@@ -108,7 +175,7 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
     @Override
     public String toString()
     {
-        return "TxnNamedRead{name='" + name + '\'' + ", key=" + key + ", update=" + get() + '}';
+        return "TxnNamedRead{name='" + name + '\'' + ", keys=" + key + ", update=" + get() + '}';
     }
 
     public int txnDataName()
@@ -116,39 +183,58 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
         return name;
     }
 
-    public PartitionKey key()
+    public Seekable key()
     {
         return key;
     }
 
-    public AsyncChain<Data> read(ConsistencyLevel consistencyLevel, Timestamp executeAt)
+    public static long nowInSeconds(Timestamp executeAt)
     {
-        SinglePartitionReadCommand command = (SinglePartitionReadCommand) get();
+        return TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc());
+    }
+
+    public AsyncChain<Data> read(ConsistencyLevel consistencyLevel, Seekable key, Timestamp executeAt)
+    {
+        ReadCommand command = get();
         if (command == null)
             return AsyncResults.success(TxnData.NOOP_DATA);
+
         // TODO (required, safety): before release, double check reasoning that this is safe
 //        AccordCommandsForKey cfk = ((SafeAccordCommandStore)safeStore).commandsForKey(key);
 //        int nowInSeconds = cfk.nowInSecondsFor(executeAt, isForWriteTxn);
         // It's fine for our nowInSeconds to lag slightly our insertion timestamp, as to the user
         // this simply looks like the transaction witnessed TTL'd data and the data then expired
         // immediately after the transaction executed, and this simplifies things a great deal
-        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc());
-        if (consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE)
-            command = command.withoutReconciliation();
-        return performLocalRead(command, nowInSeconds);
+        long nowInSeconds = nowInSeconds(executeAt);
+
+        boolean withoutReconciliation = readsWithoutReconciliation(consistencyLevel);
+        switch (key.domain())
+        {
+            case Key:
+                return performLocalKeyRead(((SinglePartitionReadCommand) command).withTransactionalSettings(withoutReconciliation, nowInSeconds));
+            case Range:
+                return performLocalRangeRead(((PartitionRangeReadCommand) command), key.asRange(), consistencyLevel, nowInSeconds);
+            default:
+                throw new IllegalStateException("Unhandled domain " + key.domain());
+        }
     }
 
+    public static boolean readsWithoutReconciliation(ConsistencyLevel consistencyLevel)
+    {
+        boolean withoutReconciliation = consistencyLevel == null || consistencyLevel == ConsistencyLevel.ONE;
+        return withoutReconciliation;
+    }
+
+
     public ReadCommand command()
     {
         return get();
     }
 
-    private AsyncChain<Data> performLocalRead(SinglePartitionReadCommand command, int nowInSeconds)
+    private AsyncChain<Data> performLocalKeyRead(SinglePartitionReadCommand read)
     {
         Callable<Data> readCallable = () ->
         {
-            SinglePartitionReadCommand read = command.withNowInSec(nowInSeconds);
-
             try (ReadExecutionController controller = read.executionController();
                  PartitionIterator iterator = UnfilteredPartitionIterators.filter(read.executeLocally(controller), read.nowInSec()))
             {
@@ -201,19 +287,132 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
                 @Override
                 public String description()
                 {
-                    return command.toCQLString();
+                    return read.toCQLString();
                 }
             }
         );
     }
 
+    public static PartitionRangeReadCommand commandForSubrange(PartitionRangeReadCommand command, Range r, ConsistencyLevel consistencyLevel, long nowInSeconds)
+    {
+        AbstractBounds<PartitionPosition> bounds = command.dataRange().keyRange();
+        PartitionPosition startPP = bounds.left;
+        PartitionPosition endPP = bounds.right;
+        TokenKey startTokenKey = new TokenKey(command.metadata().id, startPP.getToken());
+        AccordRoutingKey startRoutingKey = ((AccordRoutingKey)r.start());
+        AccordRoutingKey endRoutingKey = ((AccordRoutingKey)r.end());
+        Token subRangeStartToken = startRoutingKey.getClass() == SentinelKey.class ? startPP.getToken() : ((AccordRoutingKey)r.start()).asTokenKey().token();
+        Token subRangeEndToken = endRoutingKey.getClass() == SentinelKey.class ? endPP.getToken() : ((AccordRoutingKey)r.end()).asTokenKey().token();
+
+        /*
+         * The way ranges/bounds work for range queries is that the beginning and ending bounds from the command
+         * could be tokens (and min/max key bounds) or actual keys depending on the bounds of the top level query we
+         * are running and where we are in paging. We need to preserve whatever is in the command in case it is a
+         * key and not a token, or it's a token but might be a min/max key bound.
+         *
+         * Then Accord will further subdivide the range in the command so need to inject additional bounds in the middle
+         * that match the range ownership of Accord.
+         *
+         * The command still contains the original bound and then the Accord range passed in determines what subset of
+         * that bound we want. We have to make sure to use the bounds from the command if it is the start or end instead
+         * of a key bound created from the Accord range since it could be a real key or min/max bound.
+         *
+         * When we are dealing with a bound created by Accord's further subdivision we use a maxKeyBound (exclusive)
+         * for both beginning and end because Bounds is left and right inclusive while Range is only left inclusive.
+         * We only use TokenRange with Accord which matches the left/right inclusivity of Cassandra's Range.
+         *
+         * That means the Range we get from Accord overlaps the previous Range on the left which when converted to a Bound
+         * would potentially read the same Token twice. So the left needs to be a maxKeyBound to exclude the data that isn't
+         * owned here and to avoid potentially reading the same data twice. The right bound also needs to be a maxKeyBound since Range
+         * is right inclusive so every partition we find needs to be < the right bound.
+         */
+        boolean isFirstSubrange = startPP.getToken().equals(subRangeStartToken);
+        PartitionPosition subRangeStartPP = isFirstSubrange ? startPP : subRangeStartToken.maxKeyBound();
+        PartitionPosition subRangeEndPP = endPP.getToken().equals(subRangeEndToken) ? endPP : subRangeEndToken.maxKeyBound();
+        // Need to preserve the fact it is a bounds for paging to work, a range is not left inclusive and will not start from where we left off
+        AbstractBounds<PartitionPosition> subRange = isFirstSubrange ? bounds.withNewRight(subRangeEndPP) : new org.apache.cassandra.dht.Range(subRangeStartPP, subRangeEndPP);
+        return command.withTransactionalSettings(nowInSeconds, subRange, startTokenKey.equals(r.start()), readsWithoutReconciliation(consistencyLevel));
+    }
+
+    private AsyncChain<Data> performLocalRangeRead(PartitionRangeReadCommand command, Range r, ConsistencyLevel consistencyLevel, long nowInSeconds)
+    {
+        PartitionRangeReadCommand read = commandForSubrange(command, r, consistencyLevel, nowInSeconds);
+        Callable<Data> readCallable = () ->
+        {
+            try (ReadExecutionController controller = read.executionController();
+                 UnfilteredPartitionIterator partition = read.executeLocally(controller);
+                 PartitionIterator iterator = UnfilteredPartitionIterators.filter(partition, read.nowInSec()))
+            {
+                TxnData result = new TxnData();
+                TxnDataRangeValue value = new TxnDataRangeValue();
+                while(iterator.hasNext())
+                {
+                    try (RowIterator rows = iterator.next())
+                    {
+                        FilteredPartition filtered = FilteredPartition.create(rows);
+                        if (filtered.hasRows() || read.selectsFullPartition())
+                        {
+                            value.add(filtered);
+                        }
+                    }
+                }
+                result.put(TxnData.txnDataName(TxnDataNameKind.USER), value);
+                return result;
+            }
+        };
+
+        return AsyncChains.ofCallable(Stage.READ.executor(), readCallable, (callable, receiver) ->
+                                                                           new DebuggableTask.RunnableDebuggableTask()
+                                                                           {
+                                                                               private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
+                                                                               private volatile long approxStartTimeNanos;
+
+                                                                               @Override
+                                                                               public void run()
+                                                                               {
+                                                                                   approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
+
+                                                                                   try
+                                                                                   {
+                                                                                       Data call = callable.call();
+                                                                                       receiver.accept(call, null);
+                                                                                   }
+                                                                                   catch (Throwable t)
+                                                                                   {
+                                                                                       logger.debug("AsyncChain Callable threw an Exception", t);
+                                                                                       receiver.accept(null, t);
+                                                                                   }
+                                                                               }
+
+                                                                               @Override
+                                                                               public long creationTimeNanos()
+                                                                               {
+                                                                                   return approxCreationTimeNanos;
+                                                                               }
+
+                                                                               @Override
+                                                                               public long startTimeNanos()
+                                                                               {
+                                                                                   return approxStartTimeNanos;
+                                                                               }
+
+                                                                               @Override
+                                                                               public String description()
+                                                                               {
+                                                                                   return command.toCQLString();
+                                                                               }
+                                                                           }
+        );
+    }
+
     static final IVersionedSerializer<TxnNamedRead> serializer = new IVersionedSerializer<>()
     {
         @Override
         public void serialize(TxnNamedRead read, DataOutputPlus out, int version) throws IOException
         {
             out.writeInt(read.name);
-            PartitionKey.serializer.serialize(read.key, out, version);
+            KeySerializers.seekable.serialize(read.key, out, version);
+
             if (read.bytes() != null)
             {
                 out.write(0);
@@ -229,7 +428,7 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
         public TxnNamedRead deserialize(DataInputPlus in, int version) throws IOException
         {
             int name = in.readInt();
-            PartitionKey key = PartitionKey.serializer.deserialize(in, version);
+            Seekable key = KeySerializers.seekable.deserialize(in, version);
             ByteBuffer bytes = in.readByte() == 1 ? null : readWithVIntLength(in);
             return new TxnNamedRead(name, key, bytes);
         }
@@ -239,8 +438,8 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
         {
             long size = 0;
             size += TypeSizes.sizeof(read.name);
-            size += PartitionKey.serializer.serializedSize(read.key, version);
-            size += TypeSizes.BOOL_SIZE; // is null
+            size += KeySerializers.seekable.serializedSize(read.key, version);
+            size += TypeSizes.BYTE_SIZE; // is null
             if (read.bytes() != null)
                 size += serializedSizeWithVIntLength(read.bytes());
             return size;

src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java

@@ -29,6 +29,7 @@ import accord.api.Query;
 import accord.api.Read;
 import accord.api.Result;
 import accord.api.Update;
+import accord.primitives.Ranges;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
@@ -41,22 +42,27 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.metrics.ClientRequestsMetricsHolder;
+import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind;
 import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
-import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.ObjectSizes;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
 import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.CAS_READ;
 import static org.apache.cassandra.service.accord.txn.TxnData.txnDataName;
 
 public abstract class TxnQuery implements Query
 {
+
     /**
      * Used by transaction statements which will have Accord pass back to the C* coordinator code all the data that is
      * read even if it is not returned as part of the result to the client. TxnDataName.returning() will fetch the data
      * that is returned from TxnData.
+     *
+     * Also used by SERIAL key reads, and non-SERIAL key reads when they are executed on Accord.
      */
     public static final TxnQuery ALL = new TxnQuery()
     {
@@ -118,7 +124,7 @@ public abstract class TxnQuery implements Query
                 return new TxnData();
             else if (txnData.isEmpty())
             {
-                TxnKeyRead txnKeyRead = (TxnKeyRead)read;
+                TxnRead txnKeyRead = (TxnRead)read;
                 SinglePartitionReadCommand command = (SinglePartitionReadCommand) txnKeyRead.iterator().next().get();
                 // For CAS must return a non-empty result to indicate error even if there was no partition found
                 return TxnData.of(txnDataName(CAS_READ), new TxnDataKeyValue(EmptyIterators.row(command.metadata(), command.partitionKey(), command.isReversed())));
@@ -132,7 +138,7 @@ public abstract class TxnQuery implements Query
     /**
      * UNSAFE_EMPTY doesn't validate that the range is owned by Accord so you want to be careful and use NONE
      * if your transaction simply doesn't have results because that will validate that Accord owns the range
-     * for things like blind writes. Empty is used by Accord for things like sync points which may need to exeucte
+     * for things like blind writes. Empty is used by Accord for things like sync points which may need to execute
      * for ranges Accord used to manage, but no longer does.
      */
     public static final TxnQuery UNSAFE_EMPTY = new TxnQuery()
@@ -175,8 +181,8 @@ public abstract class TxnQuery implements Query
 
         private Result concat(TxnData data, Read read)
         {
-            TxnRangeRead txnRead = (TxnRangeRead) read;
-            PartitionRangeReadCommand command = (PartitionRangeReadCommand) txnRead.get();
+            TxnRead txnRead = (TxnRead) read;
+            PartitionRangeReadCommand command = (PartitionRangeReadCommand) txnRead.iterator().next().get();
             TxnDataRangeValue value = (TxnDataRangeValue) data.get(txnDataName(TxnDataNameKind.USER));
             Supplier<PartitionIterator> source = value.toPartitionIterator(command.isReversed());
             // Because the query was split across multiple command stores the pushed down limit won't be sufficient
@@ -200,14 +206,17 @@ public abstract class TxnQuery implements Query
     @Override
     public Result compute(TxnId txnId, Timestamp executeAt, Seekables<?, ?> keys, @Nullable Data data, @Nullable Read read, @Nullable Update update)
     {
-        Epoch epoch = Epoch.create(executeAt.epoch());
-        if (transactionIsInMigratingOrMigratedRange(epoch, keys))
+        // TODO (required): This is not the cluster metadata of the current transaction
+        ClusterMetadata clusterMetadata = ClusterMetadata.current();
+        checkState(clusterMetadata.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", clusterMetadata.epoch.getEpoch(), executeAt.epoch());
+        boolean reads = read != null && !read.keys().isEmpty();
+        if (transactionShouldBeBlocked(clusterMetadata, reads, keys))
         {
             if (txnId.isWrite())
                 ClientRequestsMetricsHolder.accordWriteMetrics.accordMigrationRejects.mark();
             else
                 ClientRequestsMetricsHolder.accordReadMetrics.accordMigrationRejects.mark();
-            return new RetryWithNewProtocolResult(epoch);
+            return RetryWithNewProtocolResult.instance;
         }
         return doCompute(txnId, executeAt, keys, data, read, update);
     }
@@ -249,19 +258,56 @@ public abstract class TxnQuery implements Query
         }
     };
 
-    private static boolean transactionIsInMigratingOrMigratedRange(Epoch epoch, Seekables<?, ?> keys)
+    private static boolean transactionShouldBeBlocked(ClusterMetadata clusterMetadata, boolean reads, Seekables<?, ?> keys)
     {
-        // TODO (required): This is going to be problematic when we presumably support range reads and don't validate them
-        // Whatever this transaction might be it isn't one supported for migration anyways
-        if (!keys.domain().isKey())
-            return false;
+        // TxnQuery needs to be smart enough to allow blind writes through for the non-transactional use cases during migration
+        // This also allows blind write TransactionStatement to run before TransactionStatemetns with reads can run,
+        // but this is harmless since we only promise that TransactionStatement works when migrated to Accord.
+        // TODO (lowpri): This could look at read keys vs write keys to see if it can run in more cases
+        if (reads)
+            return !transactionIsSafeToReadAndWrite(clusterMetadata, keys);
+        else
+            return !transactionIsSafeToWrite(clusterMetadata, keys);
+    }
+
+    private static boolean transactionIsSafeToReadAndWrite(ClusterMetadata clusterMetadata, Seekables<?, ?> keys)
+    {
+        switch (keys.domain())
+        {
+            case Key:
+                for (PartitionKey partitionKey : (Seekables<PartitionKey, ?>)keys)
+                {
+                    // TODO (required): This is looking at ClusterMetadata, but not the ClusterMetadata for the specified epoch, just that epoch or later. Need to store ConsensusMigrationState in the global Topologies Accord stores for itself.
+                    if (!ConsensusRequestRouter.instance.isKeyManagedByAccordForReadAndWrite(clusterMetadata, partitionKey.table(), partitionKey.partitionKey()))
+                        return false;
+                }
+                break;
+            case Range:
+                for (accord.primitives.Range range : (Ranges)keys)
+                {
+                    TokenRange tokenRange = (TokenRange)range;
+                    // TODO (required): This is looking at ClusterMetadata, but not the ClusterMetadata for the specified epoch, just that epoch or later. Need to store ConsensusMigrationState in the global Topologies Accord stores for itself.
+                    if (!ConsensusRequestRouter.instance.isRangeManagedByAccordForReadAndWrite(clusterMetadata, tokenRange.table(), tokenRange))
+                        return false;
+                }
+                break;
+            default:
+                throw new IllegalStateException("Unsupported domain " + keys.domain());
+        }
+
+        return true;
+    }
+
+    private static boolean transactionIsSafeToWrite(ClusterMetadata clusterMetadata, Seekables<?, ?> keys)
+    {
+        checkState(keys.domain().isKey(), "Only key transactions are supported for writes");
 
         for (PartitionKey partitionKey : (Seekables<PartitionKey, ?>)keys)
         {
             // TODO (required): This is looking at ClusterMetadata, but not the ClusterMetadata for the specified epoch, just that epoch or later. Need to store ConsensusMigrationState in the global Topologies Accord stores for itself.
-            if (ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(epoch, partitionKey.table(), partitionKey.partitionKey()))
-                return true;
+            if (!ConsensusRequestRouter.instance.isKeyManagedByAccordForWrite(clusterMetadata, partitionKey.table(), partitionKey.partitionKey()))
+                return false;
         }
-        return false;
+        return true;
     }
 }

src/java/org/apache/cassandra/service/accord/txn/TxnRangeRead.java

@@ -1,387 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord.txn;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.Nonnull;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import accord.api.Data;
-import accord.api.DataStore;
-import accord.api.Read;
-import accord.local.SafeCommandStore;
-import accord.primitives.AbstractRanges.UnionMode;
-import accord.primitives.Participants;
-import accord.primitives.Range;
-import accord.primitives.Ranges;
-import accord.primitives.Routable.Domain;
-import accord.primitives.Routables.Slice;
-import accord.primitives.Seekable;
-import accord.primitives.Seekables;
-import accord.primitives.Timestamp;
-import accord.utils.async.AsyncChain;
-import accord.utils.async.AsyncChains;
-import org.apache.cassandra.concurrent.DebuggableTask;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.db.PartitionRangeReadCommand;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.partitions.FilteredPartition;
-import org.apache.cassandra.db.partitions.PartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
-import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
-import org.apache.cassandra.db.rows.RowIterator;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.dht.Token.KeyBound;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.accord.AccordObjectSizes;
-import org.apache.cassandra.service.accord.TokenRange;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.service.accord.serializers.KeySerializers;
-import org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.MonotonicClock;
-import org.apache.cassandra.utils.ObjectSizes;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer;
-import static org.apache.cassandra.service.accord.IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS;
-import static org.apache.cassandra.utils.ByteBufferUtil.readWithVIntLength;
-import static org.apache.cassandra.utils.ByteBufferUtil.serializedSizeWithVIntLength;
-import static org.apache.cassandra.utils.ByteBufferUtil.writeWithVIntLength;
-import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable;
-import static org.apache.cassandra.utils.NullableSerializer.serializeNullable;
-import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize;
-
-public class TxnRangeRead extends AbstractSerialized<ReadCommand> implements TxnRead
-{
-    private static final Logger logger = LoggerFactory.getLogger(TxnRangeRead.class);
-
-    public static final TxnRangeRead EMPTY = new TxnRangeRead(null, null, (Ranges)null);
-    private static final long EMPTY_SIZE = ObjectSizes.measure(EMPTY);
-
-    @Nonnull
-    private final ConsistencyLevel cassandraConsistencyLevel;
-    @Nonnull
-    private final Ranges covering;
-
-    public TxnRangeRead(@Nonnull PartitionRangeReadCommand command, @Nonnull List<AbstractBounds<PartitionPosition>> ranges, @Nonnull ConsistencyLevel cassandraConsistencyLevel)
-    {
-        super(command);
-        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read");
-        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
-        TableId tableId = command.metadata().id;
-
-        TokenRange[] accordRanges = new TokenRange[ranges.size()];
-        for (int i = 0; i < ranges.size(); i++)
-        {
-            AbstractBounds<PartitionPosition> range = ranges.get(i);
-            // Should already have been unwrapped
-            checkState(!AbstractBounds.strictlyWrapsAround(range.left, range.right));
-
-            // Read commands can contain a mix of different kinds of bounds to facilitate paging
-            // and we need to communicate that to Accord as its own ranges. This uses
-            // TokenKey, SentinelKey, and MinTokenKey and sticks exclusively with left exclusive/right inclusive
-            // ranges rather add more types of ranges to the mix
-            // MinTokenKey allows emulating inclusive left and exclusive right with Range
-            boolean inclusiveLeft = range.inclusiveLeft();
-            PartitionPosition startPP = range.left;
-            boolean startIsMinKeyBound = startPP.getClass() == KeyBound.class ? ((KeyBound)startPP).isMinimumBound : false;
-            Token startToken = startPP.getToken();
-            AccordRoutingKey startAccordRoutingKey;
-            if (startToken.isMinimum() && inclusiveLeft)
-                startAccordRoutingKey = SentinelKey.min(tableId);
-            else if (inclusiveLeft || startIsMinKeyBound)
-                startAccordRoutingKey = new MinTokenKey(tableId, startToken);
-            else
-                startAccordRoutingKey = new TokenKey(tableId, startToken);
-
-            boolean inclusiveRight = range.inclusiveRight();
-            PartitionPosition endPP = range.right;
-            boolean endIsMinKeyBound = endPP.getClass() == KeyBound.class ? ((KeyBound)endPP).isMinimumBound : false;
-            Token stopToken = range.right.getToken();
-            AccordRoutingKey stopAccordRoutingKey;
-            if (stopToken.isMinimum())
-                stopAccordRoutingKey = SentinelKey.max(tableId);
-            else if (inclusiveRight && !endIsMinKeyBound)
-                stopAccordRoutingKey = new TokenKey(tableId, stopToken);
-            else
-                stopAccordRoutingKey = new MinTokenKey(tableId, stopToken);
-            accordRanges[i] = TokenRange.create(startAccordRoutingKey, stopAccordRoutingKey);
-        }
-        covering = Ranges.of(accordRanges);
-    }
-
-    private TxnRangeRead(@Nonnull ByteBuffer commandBytes, @Nonnull ConsistencyLevel cassandraConsistencyLevel, @Nonnull Ranges covering)
-    {
-        super(commandBytes);
-        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read");
-        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
-        this.covering = covering;
-    }
-
-    @Override
-    public Seekables<?, ?> keys()
-    {
-        return covering;
-    }
-
-    @Override
-    public AsyncChain<Data> read(Seekable range, SafeCommandStore commandStore, Timestamp executeAt, DataStore store)
-    {
-        // Set to null since we don't need it and interop can pass in null
-        commandStore = null;
-
-        // It's fine for our nowInSeconds to lag slightly our insertion timestamp, as to the user
-        // this simply looks like the transaction witnessed TTL'd data and the data then expired
-        // immediately after the transaction executed, and this simplifies things a great deal
-        long nowInSeconds = TimeUnit.MICROSECONDS.toSeconds(executeAt.hlc());
-
-        List<AsyncChain<Data>> results = new ArrayList<>();
-        PartitionRangeReadCommand command = (PartitionRangeReadCommand)get();
-        if (cassandraConsistencyLevel == null || cassandraConsistencyLevel == ConsistencyLevel.ONE)
-            command = command.withoutReconciliation();
-
-        Ranges intersecting = covering.slice(Ranges.of(range.asRange()), Slice.Minimal);
-        for (Range subRange : intersecting)
-            results.add(performLocalRead(command, subRange, nowInSeconds));
-        if (results.isEmpty())
-            // Result type must match everywhere
-            return AsyncChains.success(new TxnData());
-
-        return AsyncChains.reduce(results, Data::merge);
-    }
-
-    public PartitionRangeReadCommand commandForSubrange(Range r, long nowInSeconds)
-    {
-        return commandForSubrange((PartitionRangeReadCommand) get(), r, nowInSeconds);
-    }
-
-    public PartitionRangeReadCommand commandForSubrange(PartitionRangeReadCommand command, Range r, long nowInSeconds)
-    {
-        AbstractBounds<PartitionPosition> bounds = command.dataRange().keyRange();
-        PartitionPosition startPP = bounds.left;
-        PartitionPosition endPP = bounds.right;
-        TokenKey startTokenKey = new TokenKey(command.metadata().id, startPP.getToken());
-        AccordRoutingKey startRoutingKey = ((AccordRoutingKey)r.start());
-        AccordRoutingKey endRoutingKey = ((AccordRoutingKey)r.end());
-        Token subRangeStartToken = startRoutingKey.getClass() == SentinelKey.class ? startPP.getToken() : ((AccordRoutingKey)r.start()).asTokenKey().token();
-        Token subRangeEndToken = endRoutingKey.getClass() == SentinelKey.class ? endPP.getToken() : ((AccordRoutingKey)r.end()).asTokenKey().token();
-
-        /*
-         * The way ranges/bounds work for range queries is that the beginning and ending bounds from the command
-         * could be tokens (and min/max key bounds) or actual keys depending on the bounds of the top level query we
-         * are running and where we are in paging. We need to preserve whatever is in the command in case it is a
-         * key and not a token, or it's a token but might be a min/max key bound.
-         *
-         * Then Accord will further subdivide the range in the command so need to inject additional bounds in the middle
-         * that match the range ownership of Accord.
-         *
-         * The command still contains the original bound and then the Accord range passed in determines what subset of
-         * that bound we want. We have to make sure to use the bounds from the command if it is the start or end instead
-         * of a key bound created from the Accord range since it could be a real key or min/max bound.
-         *
-         * When we are dealing with a bound created by Accord's further subdivision we use a maxKeyBound (exclusive)
-         * for both beginning and end because Bounds is left and right inclusive while Range is only left inclusive.
-         * We only use TokenRange with Accord which matches the left/right inclusivity of Cassandra's Range.
-         *
-         * That means the Range we get from Accord overlaps the previous Range on the left which when converted to a Bound
-         * would potentially read the same Token twice. So the left needs to be a maxKeyBound to exclude the data that isn't
-         * owned here and to avoid potentially reading the same data twice. The right bound also needs to be a maxKeyBound since Range
-         * is right inclusive so every partition we find needs to be < the right bound.
-         */
-        boolean isFirstSubrange = startPP.getToken().equals(subRangeStartToken);
-        PartitionPosition subRangeStartPP = isFirstSubrange ? startPP : subRangeStartToken.maxKeyBound();
-        PartitionPosition subRangeEndPP = endPP.getToken().equals(subRangeEndToken) ? endPP : subRangeEndToken.maxKeyBound();
-        // Need to preserve the fact it is a bounds for paging to work, a range is not left inclusive and will not start from where we left off
-        AbstractBounds<PartitionPosition> subRange = isFirstSubrange ? bounds.withNewRight(subRangeEndPP) : new org.apache.cassandra.dht.Range(subRangeStartPP, subRangeEndPP);
-        return command.forSubRangeWithNowInSeconds(nowInSeconds, subRange, startTokenKey.equals(r.start()));
-    }
-
-    private AsyncChain<Data> performLocalRead(PartitionRangeReadCommand command, Range r, long nowInSeconds)
-    {
-        Callable<Data> readCallable = () ->
-        {
-            PartitionRangeReadCommand read = commandForSubrange(command, r, nowInSeconds);
-
-            try (ReadExecutionController controller = read.executionController();
-                 UnfilteredPartitionIterator partition = read.executeLocally(controller);
-                 PartitionIterator iterator = UnfilteredPartitionIterators.filter(partition, read.nowInSec()))
-            {
-                TxnData result = new TxnData();
-                TxnDataRangeValue value = new TxnDataRangeValue();
-                while(iterator.hasNext())
-                {
-                    try (RowIterator rows = iterator.next())
-                    {
-                        FilteredPartition filtered = FilteredPartition.create(rows);
-                        if (filtered.hasRows() || read.selectsFullPartition())
-                        {
-                            value.add(filtered);
-                        }
-                    }
-                }
-                result.put(TxnData.txnDataName(TxnDataNameKind.USER), value);
-                return result;
-            }
-        };
-
-        return AsyncChains.ofCallable(Stage.READ.executor(), readCallable, (callable, receiver) ->
-                                                                           new DebuggableTask.RunnableDebuggableTask()
-                                                                           {
-                                                                               private final long approxCreationTimeNanos = MonotonicClock.Global.approxTime.now();
-                                                                               private volatile long approxStartTimeNanos;
-
-                                                                               @Override
-                                                                               public void run()
-                                                                               {
-                                                                                   approxStartTimeNanos = MonotonicClock.Global.approxTime.now();
-
-                                                                                   try
-                                                                                   {
-                                                                                       Data call = callable.call();
-                                                                                       receiver.accept(call, null);
-                                                                                   }
-                                                                                   catch (Throwable t)
-                                                                                   {
-                                                                                       logger.debug("AsyncChain Callable threw an Exception", t);
-                                                                                       receiver.accept(null, t);
-                                                                                   }
-                                                                               }
-
-                                                                               @Override
-                                                                               public long creationTimeNanos()
-                                                                               {
-                                                                                   return approxCreationTimeNanos;
-                                                                               }
-
-                                                                               @Override
-                                                                               public long startTimeNanos()
-                                                                               {
-                                                                                   return approxStartTimeNanos;
-                                                                               }
-
-                                                                               @Override
-                                                                               public String description()
-                                                                               {
-                                                                                   return command.toCQLString();
-                                                                               }
-                                                                           }
-        );
-    }
-
-    @Override
-    public Read slice(Ranges ranges)
-    {
-        return new TxnRangeRead(bytes(), cassandraConsistencyLevel, covering.slice(ranges, Slice.Minimal));
-    }
-
-    @Override
-    public Read intersecting(Participants<?> participants)
-    {
-        checkArgument(participants.domain() == Domain.Range, "Can only intersect with ranges");
-        return new TxnRangeRead(bytes(), cassandraConsistencyLevel, covering.intersecting(participants, Slice.Minimal));
-    }
-
-    @Override
-    public Read merge(Read other)
-    {
-        Ranges newCovering = ((TxnRangeRead)other).covering.union(UnionMode.MERGE_OVERLAPPING, covering);
-        return new TxnRangeRead(bytes(), cassandraConsistencyLevel, newCovering);
-    }
-
-    @Override
-    public Kind kind()
-    {
-        return Kind.range;
-    }
-
-    @Override
-    public long estimatedSizeOnHeap()
-    {
-        long size = EMPTY_SIZE;
-        size += ByteBufferUtil.estimatedSizeOnHeap(bytes());
-        size += AccordObjectSizes.ranges(covering);
-        return size;
-    }
-
-    public PartitionRangeReadCommand command()
-    {
-        return (PartitionRangeReadCommand) get();
-    }
-
-    public static final TxnReadSerializer<TxnRangeRead> serializer = new TxnReadSerializer<TxnRangeRead>()
-    {
-        @Override
-        public void serialize(TxnRangeRead read, DataOutputPlus out, int version) throws IOException
-        {
-            serializeNullable(read.cassandraConsistencyLevel, out, version, consistencyLevelSerializer);
-            KeySerializers.ranges.serialize(read.covering, out, version);
-            writeWithVIntLength(read.bytes(), out);
-        }
-
-        @Override
-        public TxnRangeRead deserialize(DataInputPlus in, int version) throws IOException
-        {
-            ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer);
-            Ranges covering = KeySerializers.ranges.deserialize(in, version);
-            ByteBuffer commandBytes = readWithVIntLength(in);
-            return new TxnRangeRead(commandBytes, consistencyLevel, covering);
-        }
-
-        @Override
-        public long serializedSize(TxnRangeRead read, int version)
-        {
-            long size = 0;
-            size += KeySerializers.ranges.serializedSize(read.covering, version);
-            size += serializedNullableSize(read.cassandraConsistencyLevel, version, consistencyLevelSerializer);
-            size += serializedSizeWithVIntLength(read.bytes());
-            return size;
-        }
-    };
-
-    @Override
-    protected IVersionedSerializer<ReadCommand> serializer()
-    {
-        return PartitionRangeReadCommand.serializer;
-    }
-
-    @Override
-    public ConsistencyLevel cassandraConsistencyLevel()
-    {
-        return cassandraConsistencyLevel;
-    }
-}

src/java/org/apache/cassandra/service/accord/txn/TxnRead.java

@@ -19,75 +19,321 @@
 package org.apache.cassandra.service.accord.txn;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
+import com.google.common.collect.ImmutableList;
+
+import accord.api.Data;
+import accord.api.DataStore;
+import accord.api.Key;
 import accord.api.Read;
+import accord.local.SafeCommandStore;
+import accord.primitives.Keys;
+import accord.primitives.Participants;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
+import accord.primitives.Seekable;
+import accord.primitives.Seekables;
+import accord.primitives.Timestamp;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
 import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.ObjectSizes;
 
-import static org.apache.cassandra.db.TypeSizes.sizeof;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer;
+import static org.apache.cassandra.service.accord.IAccordService.SUPPORTED_READ_CONSISTENCY_LEVELS;
+import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.CAS_READ;
+import static org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind.USER;
+import static org.apache.cassandra.service.accord.txn.TxnData.txnDataName;
+import static org.apache.cassandra.utils.ArraySerializers.deserializeArray;
+import static org.apache.cassandra.utils.ArraySerializers.serializeArray;
+import static org.apache.cassandra.utils.ArraySerializers.serializedArraySize;
+import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable;
+import static org.apache.cassandra.utils.NullableSerializer.serializeNullable;
+import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize;
 
-public interface TxnRead extends Read
+public class TxnRead extends AbstractKeySorted<TxnNamedRead> implements Read
 {
-    ConsistencyLevel cassandraConsistencyLevel();
+    private static final TxnRead EMPTY_KEY = new TxnRead(Domain.Key);
+    private static final TxnRead EMPTY_RANGE = new TxnRead(Domain.Range);
+    private static final long EMPTY_SIZE = ObjectSizes.measure(EMPTY_KEY);
+    private static final Comparator<TxnNamedRead> TXN_NAMED_READ_KEY_COMPARATOR = Comparator.comparing(a -> ((PartitionKey) a.key()));
+    private static final byte TYPE_EMPTY_KEY = 0;
+    private static final byte TYPE_EMPTY_RANGE = 1;
+    private static final byte TYPE_NOT_EMPTY = 2;
 
-    public interface TxnReadSerializer<T extends TxnRead> extends IVersionedSerializer<T> {}
-
-    enum Kind
+    public static TxnRead empty(Domain domain)
     {
-        key(0),
-        range(1);
-
-        int id;
-
-        Kind(int id)
+        switch (domain)
         {
-            this.id = id;
-        }
-
-        public TxnReadSerializer serializer()
-        {
-            switch (this)
-            {
-                case key:
-                    return TxnKeyRead.serializer;
-                case range:
-                    return TxnRangeRead.serializer;
-                default:
-                    throw new IllegalStateException("Unrecognized kind " + this);
-            }
+            default:
+                throw new IllegalStateException("Unhandled domain " + domain);
+            case Key:
+                return EMPTY_KEY;
+            case Range:
+                return EMPTY_RANGE;
         }
     }
 
-    long estimatedSizeOnHeap();
+    // Cassandra's consistency level used by Accord to safely read data written outside of Accord
+    @Nullable
+    private final ConsistencyLevel cassandraConsistencyLevel;
 
-    Kind kind();
+    // Specifies the domain in case the TxnRead is empty and it can't be inferred
+    private final Domain domain;
 
-    void unmemoize();
-
-    IVersionedSerializer<TxnRead> serializer = new IVersionedSerializer<TxnRead>()
+    private TxnRead(Domain domain)
     {
-        @SuppressWarnings("unchecked")
-        @Override
-        public void serialize(TxnRead txnRead, DataOutputPlus out, int version) throws IOException
+        super(new TxnNamedRead[0], domain);
+        this.domain = domain;
+        this.cassandraConsistencyLevel = null;
+    }
+
+    private TxnRead(@Nonnull TxnNamedRead[] items, @Nullable ConsistencyLevel cassandraConsistencyLevel)
+    {
+        super(items, items[0].key().domain());
+        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read: %s", cassandraConsistencyLevel);
+        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
+        this.domain = items[0].key().domain();
+    }
+
+    private TxnRead(@Nonnull List<TxnNamedRead> items, @Nullable ConsistencyLevel cassandraConsistencyLevel)
+    {
+        super(items, items.get(0).key().domain());
+        checkArgument(cassandraConsistencyLevel == null || SUPPORTED_READ_CONSISTENCY_LEVELS.contains(cassandraConsistencyLevel), "Unsupported consistency level for read: %s", cassandraConsistencyLevel);
+        this.cassandraConsistencyLevel = cassandraConsistencyLevel;
+        this.domain = items.get(0).key().domain();
+    }
+
+    private static void sortReads(List<TxnNamedRead> reads)
+    {
+        if (reads.size() == 0)
+            return;
+        reads.sort(TXN_NAMED_READ_KEY_COMPARATOR);
+    }
+
+    public static TxnRead createTxnRead(@Nonnull List<TxnNamedRead> items, @Nullable ConsistencyLevel consistencyLevel, Domain domain)
+    {
+        if (items.isEmpty())
+            return empty(domain);
+        sortReads(items);
+        return new TxnRead(items, consistencyLevel);
+    }
+
+    public static TxnRead createSerialRead(List<SinglePartitionReadCommand> readCommands, ConsistencyLevel consistencyLevel)
+    {
+        List<TxnNamedRead> reads = new ArrayList<>(readCommands.size());
+        for (int i = 0; i < readCommands.size(); i++)
+            reads.add(new TxnNamedRead(txnDataName(USER, i), readCommands.get(i)));
+        sortReads(reads);
+        return new TxnRead(reads, consistencyLevel);
+    }
+
+    public static TxnRead createCasRead(SinglePartitionReadCommand readCommand, ConsistencyLevel consistencyLevel)
+    {
+        TxnNamedRead read = new TxnNamedRead(txnDataName(CAS_READ), readCommand);
+        return new TxnRead(ImmutableList.of(read), consistencyLevel);
+    }
+
+    // A read that declares it will read from keys but doesn't actually read any data so dependent transactions will
+    // still be applied first
+    public static TxnRead createNoOpRead(Keys keys)
+    {
+        List<TxnNamedRead> reads = new ArrayList<>(keys.size());
+        for (int i = 0; i < keys.size(); i++)
+            reads.add(new TxnNamedRead(txnDataName(USER, i), keys.get(i), null));
+        return new TxnRead(reads, null);
+    }
+
+    public static TxnRead createRangeRead(PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> range, ConsistencyLevel consistencyLevel)
+    {
+        return new TxnRead(ImmutableList.of(new TxnNamedRead(txnDataName(USER), range, command)), consistencyLevel);
+    }
+
+    public long estimatedSizeOnHeap()
+    {
+        long size = EMPTY_SIZE;
+        for (TxnNamedRead read : items)
+            size += read.estimatedSizeOnHeap();
+        return size;
+    }
+
+    @Override
+    int compareNonKeyFields(TxnNamedRead left, TxnNamedRead right)
+    {
+        return Integer.compare(left.txnDataName(), right.txnDataName());
+    }
+
+    @Override
+    Seekable getKey(TxnNamedRead read)
+    {
+        return read.key();
+    }
+
+    @Override
+    TxnNamedRead[] newArray(int size)
+    {
+        return new TxnNamedRead[size];
+    }
+
+    @Override
+    public Seekables<?, ?> keys()
+    {
+        return itemKeys;
+    }
+
+    @Override
+    public Domain domain()
+    {
+        return domain;
+    }
+
+    public ConsistencyLevel cassandraConsistencyLevel()
+    {
+        return cassandraConsistencyLevel;
+    }
+
+    @Override
+    public Read slice(Ranges ranges)
+    {
+        return intersecting(itemKeys.slice(ranges));
+    }
+
+    @Override
+    public Read intersecting(Participants<?> participants)
+    {
+        return intersecting(itemKeys.intersecting(participants));
+    }
+
+    private Read intersecting(Seekables<?, ?> select)
+    {
+        // TODO (review): Why construct this keys at all and not just check against select?
+        Seekables<?, ?> keys = (Seekables<?, ?>)itemKeys.intersecting(select);
+        List<TxnNamedRead> reads = new ArrayList<>(keys.size());
+
+        switch (keys.domain())
         {
-            out.writeByte(txnRead.kind().ordinal());
-            txnRead.kind().serializer().serialize(txnRead, out, version);
+            case Key:
+                for (TxnNamedRead read : items)
+                    if (keys.contains((Key)read.key()))
+                        reads.add(read);
+                break;
+            case Range:
+                for (TxnNamedRead read : items)
+                    if (keys.intersects((Range)read.key()))
+                        reads.add(read);
+                break;
+            default:
+                throw new IllegalStateException("Unhandled domain " + keys.domain());
+        }
+
+        return createTxnRead(reads, cassandraConsistencyLevel, keys.domain());
+    }
+
+    @Override
+    public Read merge(Read read)
+    {
+        TxnRead txnRead = (TxnRead)read;
+        List<TxnNamedRead> reads = new ArrayList<>(items.length);
+        Collections.addAll(reads, items);
+
+        for (TxnNamedRead namedRead : txnRead)
+            if (!reads.contains(namedRead))
+                reads.add(namedRead);
+
+        return createTxnRead(reads, cassandraConsistencyLevel, txnRead.domain);
+    }
+
+    public void unmemoize()
+    {
+        for (TxnNamedRead read : items)
+            read.unmemoize();
+    }
+
+    @Override
+    public AsyncChain<Data> read(Seekable key, SafeCommandStore safeStore, Timestamp executeAt, DataStore store)
+    {
+        // Set to null since we don't need it and interop can pass in null
+        safeStore = null;
+        ClusterMetadata cm = ClusterMetadata.current();
+        checkState(cm.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", cm.epoch.getEpoch(), executeAt.epoch());
+
+        List<AsyncChain<Data>> results = new ArrayList<>();
+        forEachWithKey(key, read -> results.add(read.read(cassandraConsistencyLevel, key, executeAt)));
+
+        if (results.isEmpty())
+            // Result type must match everywhere
+            return AsyncChains.success(new TxnData());
+
+        if (results.size() == 1)
+            return results.get(0);
+
+        return AsyncChains.reduce(results, Data::merge);
+    }
+
+    public static final IVersionedSerializer<TxnRead> serializer = new IVersionedSerializer<TxnRead>()
+    {
+        @Override
+        public void serialize(TxnRead read, DataOutputPlus out, int version) throws IOException
+        {
+            if (read.items.length > 0)
+            {
+                out.write(TYPE_NOT_EMPTY);
+                serializeArray(read.items, out, version, TxnNamedRead.serializer);
+                serializeNullable(read.cassandraConsistencyLevel, out, version, consistencyLevelSerializer);
+            }
+            else
+            {
+                out.write(read.domain == Domain.Key ? TYPE_EMPTY_KEY : TYPE_EMPTY_RANGE);
+            }
         }
 
         @Override
         public TxnRead deserialize(DataInputPlus in, int version) throws IOException
         {
-            TxnRead.Kind kind = TxnRead.Kind.values()[in.readByte()];
-            return (TxnRead)kind.serializer().deserialize(in, version);
+            byte type = in.readByte();
+            switch (type)
+            {
+                default:
+                    throw new IllegalStateException("Unhandled type " + type);
+                case TYPE_EMPTY_KEY:
+                    return EMPTY_KEY;
+                case TYPE_EMPTY_RANGE:
+                    return EMPTY_RANGE;
+                case TYPE_NOT_EMPTY:
+                    TxnNamedRead[] items = deserializeArray(in, version, TxnNamedRead.serializer, TxnNamedRead[]::new);
+                    ConsistencyLevel consistencyLevel = deserializeNullable(in, version, consistencyLevelSerializer);
+                    return new TxnRead(items, consistencyLevel);
+            }
         }
 
-        @SuppressWarnings("unchecked")
         @Override
-        public long serializedSize(TxnRead txnRead, int version)
+        public long serializedSize(TxnRead read, int version)
         {
-            return sizeof((byte)txnRead.kind().ordinal()) + txnRead.kind().serializer().serializedSize(txnRead, version);
+            long size = 1; // type
+            if (read.items.length > 0)
+            {
+                size += serializedArraySize(read.items, version, TxnNamedRead.serializer);
+                size += serializedNullableSize(read.cassandraConsistencyLevel, version, consistencyLevelSerializer);
+            }
+            return size;
         }
     };
 }

src/java/org/apache/cassandra/service/accord/txn/TxnUpdate.java

@@ -49,12 +49,14 @@ import org.apache.cassandra.service.accord.AccordObjectSizes;
 import org.apache.cassandra.service.accord.AccordSerializers;
 import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 
 import static accord.utils.Invariants.requireArgument;
 import static accord.utils.SortedArrays.Search.CEIL;
+import static com.google.common.base.Preconditions.checkState;
 import static org.apache.cassandra.service.accord.AccordSerializers.consistencyLevelSerializer;
 import static org.apache.cassandra.service.accord.AccordSerializers.serialize;
 import static org.apache.cassandra.utils.ArraySerializers.deserializeArray;
@@ -214,6 +216,8 @@ public class TxnUpdate extends AccordUpdate
     @Override
     public TxnWrite apply(Timestamp executeAt, Data data)
     {
+        ClusterMetadata cm = ClusterMetadata.current();
+        checkState(cm.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", cm.epoch.getEpoch(), executeAt.epoch());
         if (!checkCondition(data))
             return TxnWrite.EMPTY_CONDITION_FAILED;
 

src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java

@@ -26,9 +26,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,6 +35,7 @@ import accord.api.DataStore;
 import accord.api.Write;
 import accord.local.SafeCommandStore;
 import accord.primitives.PartialTxn;
+import accord.primitives.Routable.Domain;
 import accord.primitives.RoutableKey;
 import accord.primitives.Seekable;
 import accord.primitives.Timestamp;
@@ -50,6 +49,7 @@ import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.Columns;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
@@ -58,11 +58,14 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.service.accord.AccordObjectSizes;
 import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.BooleanSerializer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 
+import static com.google.common.base.Preconditions.checkState;
 import static org.apache.cassandra.service.accord.AccordSerializers.partitionUpdateSerializer;
 import static org.apache.cassandra.utils.ArraySerializers.deserializeArray;
 import static org.apache.cassandra.utils.ArraySerializers.serializeArray;
@@ -100,7 +103,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
         long estimatedSizeOnHeap()
         {
             return EMPTY_SIZE
-                   + key.estimatedSizeOnHeap()
+                   + AccordObjectSizes.key(key)
                    + ByteBufferUtil.estimatedSizeOnHeap(bytes());
         }
 
@@ -135,7 +138,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
             PartitionUpdate update = get();
             if (!preserveTimestamps)
                 update = new PartitionUpdate.Builder(get(), 0).updateAllTimestamp(timestamp).build();
-            Mutation mutation = new Mutation(update, true);
+            Mutation mutation = new Mutation(update, PotentialTxnConflicts.ALLOW);
             return AsyncChains.ofRunnable(Stage.MUTATION.executor(), mutation::applyUnsafe);
         }
 
@@ -274,7 +277,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
 
         private static RegularAndStaticColumns columns(PartitionUpdate update, TxnReferenceOperations referenceOps)
         {
-            Preconditions.checkState(!referenceOps.isEmpty());
+            checkState(!referenceOps.isEmpty());
             RegularAndStaticColumns current = update.columns();
             return new RegularAndStaticColumns(columns(current.statics, referenceOps.statics),
                                                columns(current.regulars, referenceOps.regulars));
@@ -287,7 +290,7 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
 
             if (existing != null && !existing.isEmpty())
             {
-                Preconditions.checkState(existing.clustering().equals(clustering));
+                checkState(existing.clustering().equals(clustering));
                 up.addRow(existing);
             }
             else
@@ -335,13 +338,13 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
 
     private TxnWrite(Update[] items, boolean isConditionMet)
     {
-        super(items);
+        super(items, Domain.Key);
         this.isConditionMet = isConditionMet;
     }
 
     public TxnWrite(List<Update> items, boolean isConditionMet)
     {
-        super(items);
+        super(items, Domain.Key);
         this.isConditionMet = isConditionMet;
     }
 
@@ -352,11 +355,17 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
     }
 
     @Override
-    PartitionKey getKey(Update item)
+    Seekable getKey(Update item)
     {
         return item.key;
     }
 
+    @Override
+    Domain domain()
+    {
+        return Domain.Key;
+    }
+
     @Override
     Update[] newArray(int size)
     {
@@ -372,6 +381,8 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
     @Override
     public AsyncChain<Void> apply(Seekable key, SafeCommandStore safeStore, TxnId txnId, Timestamp executeAt, DataStore store, PartialTxn txn)
     {
+        ClusterMetadata cm = ClusterMetadata.current();
+        checkState(cm.epoch.getEpoch() >= executeAt.epoch(), "TCM epoch %d is < executeAt epoch %d", cm.epoch.getEpoch(), executeAt.epoch());
         // UnrecoverableRepairUpdate will deserialize as null at other nodes
         // Accord should skip the Update for a read transaction, but handle it here anyways
         TxnUpdate txnUpdate = ((TxnUpdate)txn.update());
@@ -379,7 +390,6 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
             return Writes.SUCCESS;
 
         long timestamp = executeAt.uniqueHlc();
-        int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestamp);
 
         // TODO (expected): optimise for the common single update case; lots of lists allocated
         List<AsyncChain<Void>> results = new ArrayList<>();

src/java/org/apache/cassandra/service/consensus/TransactionalMode.java

@@ -18,11 +18,9 @@
 
 package org.apache.cassandra.service.consensus;
 
-import com.google.common.collect.ImmutableList;
-
 import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.dht.NormalizedRanges;
-import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.IAccordService;
@@ -32,7 +30,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.LocalizeString;
 
 import static com.google.common.base.Preconditions.checkState;
-import static org.apache.cassandra.dht.NormalizedRanges.normalizedRanges;
 
 /*
  * Configure the transactional behavior of a table. Enables accord on a table and defines how it mixes with non-serial writes
@@ -70,33 +67,7 @@ import static org.apache.cassandra.dht.NormalizedRanges.normalizedRanges;
 public enum TransactionalMode
 {
     // Running on Paxos V1 or V2 with Accord disabled
-    off(false, false, false, false, false),
-
-    // TODO (maybe): These unsafe modes don't have Accord do async commit and single replica reads so how useful are they besides preserving non-SERIAL performance?
-    // These modes are unsafe when Accord and non-SERIAL reads and writes interact with the same data
-    // They don't guarantee that non-SERIAL reads or writes will see the latest Accord writes or that
-    // Accord transactions will recover correctly
-
-    /*
-     * Enables Accord but does not allow non-SERIAL reads and writes to occur safely to data read/written by Accord
-     *
-     * Execute non-SERIAL writes through Cassandra via StorageProxy's normal write path. This can lead Accord to compute
-     * multiple outcomes for a transaction that depend on data written by non-SERIAL writes.
-     *
-     * SERIAL reads and CAS will run on Accord. Accord will honor provided consistency levels and do synchronous commit
-     * so the results can be read correctly with non-SERIAL CLs, but read repair could interfere with Accord.
-     */
-    unsafe(true, false, false, false, false),
-
-    /*
-     * Enables Accord and makes it safe to perform non-SERIAL reads of Accord data without guaranteeing that they will
-     * see the latest Accord writes. non-SERIAL writes to data read by Accord will make Accord txn recovery non-deterministic
-     *
-     * Allow mixing of non-SERIAL writes and Accord, but still force BRR through Accord.
-     * This mode makes it safe to perform non-SERIAL or SERIAL reads of Accord data, but unsafe
-     * to write data that Accord may attempt to read.
-     */
-    unsafe_writes(true, false, false, false, true),
+    off(false, false, false, false),
 
     // These modes always provide correct execution with mixed_reads allow non-transaction non-SERIAL operations
     // at the expense of slower Accord transactions, and full allowing faster transaction execution, but forcing
@@ -107,37 +78,68 @@ public enum TransactionalMode
      * writes at the provided consistency level so they can be read via non-SERIAL consistency levels.
      * This mode makes it safe to read/write data that Accord will read/write.
      */
-    mixed_reads(true, false, true, false, true),
+    mixed_reads(true, true, false, true),
 
     /*
      * Execute writes through Accord skipping StorageProxy's normal write path. Ignores the provided consistency level
      * which makes Accord commit writes at ANY similar to Paxos with commit consistency level ANY.
      */
-    full(true, true, true, true, true),
+    full(true, true, true, true),
+
+    // TODO (maybe): These unsafe modes don't have Accord do async commit and single replica reads so how useful are they besides preserving non-SERIAL performance?
+    // These modes are unsafe when Accord and non-SERIAL reads and writes interact with the same data
+    // They don't guarantee that non-SERIAL reads or writes will see the latest Accord writes or that
+    // Accord transactions will recover correctly
+
+    /*
+     * Enables Accord and makes it safe to perform non-SERIAL reads of Accord data without guaranteeing that they will
+     * see the latest Accord writes. non-SERIAL writes to data read by Accord will make Accord txn recovery non-deterministic
+     *
+     * Allow mixing of non-SERIAL writes and Accord, but still force BRR through Accord.
+     * This mode makes it safe to perform non-SERIAL or SERIAL reads of Accord data, but unsafe
+     * to write data that Accord may attempt to read.
+     */
+    test_unsafe_writes(true, false, false, true),
+
+    /*
+     * Enables Accord but does not allow non-SERIAL reads and writes to occur safely to data read/written by Accord
+     *
+     * Execute non-SERIAL writes through Cassandra via StorageProxy's normal write path. This can lead Accord to compute
+     * multiple outcomes for a transaction that depend on data written by non-SERIAL writes.
+     *
+     * SERIAL reads and CAS will run on Accord. Accord will honor provided consistency levels and do synchronous commit
+     * so the results can be read correctly with non-SERIAL CLs, but read repair could interfere with Accord.
+     **/
+    test_unsafe(true, false, false, false),
 
     // For tests, Accord will read and be forced to do interop reads
-    test_interop_read(true, false, false, true, true);
+    test_interop_read(true, false, true, true);
 
     public final boolean accordIsEnabled;
-    public final boolean ignoresSuppliedCommitCL;
     public final boolean nonSerialWritesThroughAccord;
     public final boolean nonSerialReadsThroughAccord;
     public final boolean blockingReadRepairThroughAccord;
     private final String cqlParam;
 
-    TransactionalMode(boolean accordIsEnabled, boolean ignoreSuppliedCommitCL, boolean nonSerialWritesThroughAccord, boolean nonSerialReadsThroughAccord, boolean blockingReadRepairThroughAccord)
+    TransactionalMode(boolean accordIsEnabled, boolean nonSerialWritesThroughAccord, boolean nonSerialReadsThroughAccord, boolean blockingReadRepairThroughAccord)
     {
         this.accordIsEnabled = accordIsEnabled;
-        this.ignoresSuppliedCommitCL = ignoreSuppliedCommitCL;
         this.nonSerialWritesThroughAccord = nonSerialWritesThroughAccord;
         this.nonSerialReadsThroughAccord = nonSerialReadsThroughAccord;
         this.blockingReadRepairThroughAccord = blockingReadRepairThroughAccord;
         this.cqlParam = String.format("transactional_mode = '%s'", LocalizeString.toLowerCaseLocalized(this.name()));
+        checkState(this.name().startsWith("test_") || (nonSerialReadsThroughAccord && nonSerialWritesThroughAccord) || !nonSerialReadsThroughAccord, "Doesn't make sense to do non-SERIAL reads through Accord without also doing non-SERIAL writes through Accord");
     }
 
-    public ConsistencyLevel commitCLForStrategy(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, Token token)
+    // This can be inferred from whether non-SERIAL reads are done through Accord
+    public boolean ignoresSuppliedCommitCL()
     {
-        if (ignoresSuppliedCommitCL)
+        return nonSerialReadsThroughAccord;
+    }
+
+    public ConsistencyLevel commitCLForMode(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, Token token)
+    {
+        if (ignoresSuppliedCommitCL())
         {
             TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId);
             checkState(tms != null || fromMode == TransactionalMigrationFromMode.none);
@@ -160,13 +162,12 @@ public enum TransactionalMode
      * Infer whether Accord can ignore the read CL and bias towards correctness by reading from a quorum
      * if it's needed due to how non-SERIAL writes are done
      */
-    private boolean ignoresSuppliedReadCL()
+    public boolean ignoresSuppliedReadCL()
     {
         return nonSerialWritesThroughAccord && blockingReadRepairThroughAccord;
     }
 
-
-    public ConsistencyLevel readCLForStrategy(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, Token token)
+    public ConsistencyLevel readCLForMode(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, Token token)
     {
         if (ignoresSuppliedReadCL())
         {
@@ -187,19 +188,18 @@ public enum TransactionalMode
         return consistencyLevel;
     }
 
-    public ConsistencyLevel readCLForStrategy(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, Range<Token> range)
+    public ConsistencyLevel readCLForMode(TransactionalMigrationFromMode fromMode, ConsistencyLevel consistencyLevel, ClusterMetadata cm, TableId tableId, AbstractBounds<PartitionPosition> range)
     {
         if (ignoresSuppliedReadCL())
         {
             TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId);
             checkState(tms != null || fromMode == TransactionalMigrationFromMode.none);
 
-            NormalizedRanges<Token> ranges = normalizedRanges(ImmutableList.of(range));
             // Only ignore the supplied consistency level if none of the range is migrating
             // otherwise honor it because we might read through Accord for non-SERIAL reads before repair is run
             // this is OK to do because BRR still works and Accord isn't computing a write so recovery
             // determinism isn't an issue
-            if (tms == null || tms.migratedRanges.intersection(ranges).equals(ranges))
+            if (tms == null || range.intersects(tms.migratedRangesAsPartitionPosition()))
                 return null;
         }
 
@@ -232,4 +232,9 @@ public enum TransactionalMode
     {
         return valueOf(LocalizeString.toLowerCaseLocalized(name));
     }
+
+    public boolean isTestMode()
+    {
+        return name().startsWith("test_");
+    }
 }

src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java

@@ -20,17 +20,21 @@ package org.apache.cassandra.service.consensus.migration;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.SortedSet;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.api.BarrierType;
+import accord.primitives.Keys;
 import accord.primitives.Seekables;
 import com.github.benmanes.caffeine.cache.CacheLoader;
 import com.github.benmanes.caffeine.cache.Caffeine;
@@ -66,6 +70,7 @@ import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Collectors3;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.UUIDSerializer;
@@ -176,7 +181,7 @@ public abstract class ConsensusKeyMigrationState
             repairKeyAccord(key, tms.tableId, tms.minMigrationEpoch(key.getToken()).getEpoch(), Dispatcher.RequestTime.forImmediateExecution(), false, isForWrite);
         }
 
-        private boolean paxosReadSatisfiedByKeyMigration()
+        boolean paxosReadSatisfiedByKeyMigration()
         {
             // No migration in progress, it's safe
             if (tableMigrationState == null)
@@ -230,12 +235,6 @@ public abstract class ConsensusKeyMigrationState
         saveConsensusKeyMigrationLocally(key, tableUUID, migratedAt);
     }
 
-    /*
-     * Should be called where we know we replicate the key so that the system table contains useful information
-     * about whether the migration already occurred.
-     *
-     * This is a more expensive check that might read from the system table to determine if migration occurred.
-     */
     public static KeyMigrationState getKeyMigrationState(TableId tableId, DecoratedKey key)
     {
         ClusterMetadata cm = ClusterMetadata.current();
@@ -243,10 +242,20 @@ public abstract class ConsensusKeyMigrationState
         // No state means no migration for this table
         if (tms == null)
             return KeyMigrationState.MIGRATION_NOT_NEEDED;
+        return getKeyMigrationState(cm, tms, key);
+    }
 
+    /*
+     * Should be called where we know we replicate the key so that the system table contains useful information
+     * about whether the migration already occurred.
+     *
+     * This is a more expensive check that might read from the system table to determine if migration occurred.
+     */
+    static KeyMigrationState getKeyMigrationState(ClusterMetadata cm, TableMigrationState tms, DecoratedKey key)
+    {
         if (tms.migratingRanges.intersects(key.getToken()))
         {
-            ConsensusMigratedAt consensusMigratedAt = getConsensusMigratedAt(tableId, key);
+            ConsensusMigratedAt consensusMigratedAt = getConsensusMigratedAt(tms.tableId, key);
             if (consensusMigratedAt == null)
                 return new KeyMigrationState(null, cm.epoch, tms, key);
             return new KeyMigrationState(consensusMigratedAt, cm.epoch, tms, key);
@@ -269,6 +278,16 @@ public abstract class ConsensusKeyMigrationState
                                 Dispatcher.RequestTime requestTime,
                                 boolean global,
                                 boolean isForWrite)
+    {
+        repairKeysAccord(ImmutableList.of(key), tableId, minEpoch, requestTime, global, isForWrite);
+    }
+
+    static void repairKeysAccord(List<DecoratedKey> keys,
+                                 TableId tableId,
+                                 long minEpoch,
+                                 Dispatcher.RequestTime requestTime,
+                                 boolean global,
+                                 boolean isForWrite)
     {
         ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
         if (isForWrite)
@@ -282,7 +301,8 @@ public abstract class ConsensusKeyMigrationState
             // will soon be ready to execute, but only waits for the local replica to be ready
             // Local will only create a transaction if it can't find an existing one to wait on
             BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local;
-            Seekables keysOrRanges = AccordService.instance().barrier(Seekables.of(new PartitionKey(tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
+            SortedSet<PartitionKey> partitionKeys = keys.stream().map(key -> new PartitionKey(tableId, key)).collect(Collectors3.toSortedSet());
+            Seekables keysOrRanges = AccordService.instance().barrier(new Keys(partitionKeys), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
             if (keysOrRanges.isEmpty())
                 throw new RetryOnDifferentSystemException();
             // We don't save the state to the cache here. Accord will notify the agent every time a barrier happens.

src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationMutationHelper.java

@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
 
 import accord.coordinate.CoordinationFailed;
 import accord.primitives.Keys;
+import accord.primitives.Routable.Domain;
 import accord.primitives.Txn;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -54,8 +55,8 @@ import org.apache.cassandra.service.accord.IAccordService;
 import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.txn.TxnCondition;
-import org.apache.cassandra.service.accord.txn.TxnKeyRead;
 import org.apache.cassandra.service.accord.txn.TxnQuery;
+import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnReferenceOperations;
 import org.apache.cassandra.service.accord.txn.TxnUpdate;
 import org.apache.cassandra.service.accord.txn.TxnWrite;
@@ -115,10 +116,10 @@ public class ConsensusMigrationMutationHelper
                 TableParams tableParams = getTableMetadata(cm, tableId).params;
                 TransactionalMode mode = tableParams.transactionalMode;
                 TransactionalMigrationFromMode migrationFromMode = tableParams.transactionalMigrationFrom;
-                // commitCLForStrategy should return either null or the supplied consistency level
+                // commitCLForMode should return either null or the supplied consistency level
                 // in which case we will commit everything at that CL since Accord doesn't support per table
                 // commit consistency
-                ConsistencyLevel commitCL = mode.commitCLForStrategy(migrationFromMode, consistencyLevel, cm, tableId, mutation.key().getToken());
+                ConsistencyLevel commitCL = mode.commitCLForMode(migrationFromMode, consistencyLevel, cm, tableId, mutation.key().getToken());
                 if (commitCL != null)
                     return commitCL;
             }
@@ -217,11 +218,11 @@ public class ConsensusMigrationMutationHelper
 
     public <T extends IMutation> SplitMutation<T> splitMutationIntoAccordAndNormal(T mutation, ClusterMetadata cm)
     {
-        if (mutation.allowsPotentialTransactionConflicts())
+        if (mutation.potentialTxnConflicts().allowed)
             return new SplitMutation<>(null, mutation);
 
         Token token = mutation.key().getToken();
-        Predicate<TableId> isAccordUpdate = tableId -> tokenShouldBeWrittenThroughAccord(cm, tableId, token, TransactionalMode::nonSerialWritesThroughAccord, TransactionalMigrationFromMode::nonSerialReadsThroughAccord);
+        Predicate<TableId> isAccordUpdate = tableId -> tokenShouldBeWrittenThroughAccord(cm, tableId, token, TransactionalMode::nonSerialWritesThroughAccord, TransactionalMigrationFromMode::nonSerialWritesThroughAccord);
 
         T accordMutation = (T)mutation.filter(isAccordUpdate);
         T normalMutation = (T)mutation.filter(not(isAccordUpdate));
@@ -258,15 +259,15 @@ public class ConsensusMigrationMutationHelper
         // Potentially ignore commit consistency level if the TransactionalMode specifies full
         ConsistencyLevel clForCommit = consistencyLevelForCommit(cm, mutations, consistencyLevel);
         TxnUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit, true);
-        Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnKeyRead.EMPTY, TxnQuery.NONE, update);
+        Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.empty(Domain.Key), TxnQuery.NONE, update);
         IAccordService accordService = AccordService.instance();
         try
         {
-            return accordService.coordinateAsync(minEpoch, txn, consistencyLevel, requestTime);
+            return accordService.coordinateAsync(minEpoch, txn, clForCommit, requestTime);
         }
         catch (CoordinationFailed coordinationFailed)
         {
-            AsyncTxnResult failure = new AsyncTxnResult(coordinationFailed.txnId());
+            AsyncTxnResult failure = new AsyncTxnResult(coordinationFailed.txnId(), minEpoch, clForCommit, true, requestTime);
             failure.setFailure(coordinationFailed.wrap());
             return failure;
         }
@@ -274,15 +275,16 @@ public class ConsensusMigrationMutationHelper
 
     public static void validateSafeToExecuteNonTransactionally(IMutation mutation) throws RetryOnDifferentSystemException
     {
-        if (mutation.allowsPotentialTransactionConflicts())
+        if (mutation.potentialTxnConflicts().allowed)
             return;
 
+        String keyspace = mutation.getKeyspaceName();
         // System keyspaces are never managed by Accord
-        if (SchemaConstants.isSystemKeyspace(mutation.getKeyspaceName()))
+        if (SchemaConstants.isSystemKeyspace(keyspace))
             return;
 
         // Local keyspaces are never managed by Accord
-        if (Schema.instance.localKeyspaces().containsKeyspace(mutation.getKeyspaceName()))
+        if (Schema.instance.localKeyspaces().containsKeyspace(keyspace))
             return;
 
         ClusterMetadata cm = ClusterMetadata.current();
@@ -297,7 +299,7 @@ public class ConsensusMigrationMutationHelper
         {
             TableId tableId = pu.metadata().id;
             ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
-            if (tokenShouldBeWrittenThroughAccord(cm, tableId, dk.getToken(), TransactionalMode::nonSerialWritesThroughAccord, TransactionalMigrationFromMode::nonSerialReadsThroughAccord))
+            if (tokenShouldBeWrittenThroughAccord(cm, tableId, dk.getToken(), TransactionalMode::nonSerialWritesThroughAccord, TransactionalMigrationFromMode::nonSerialWritesThroughAccord))
             {
                 throwRetryOnDifferentSystem = true;
                 if (markedColumnFamilies == null)
@@ -350,21 +352,18 @@ public class ConsensusMigrationMutationHelper
             // with different results if Accord reads non-transactionally written data that could be seen differently by different coordinators
 
             // If the current mode writes through Accord then we should always write though Accord for ranges managed by Accord.
-            // Accord needs to do synchronous commit and respect the consistency level so that Accord will later be able to
-            // read its own writes
+            // Accord needs to do synchronous commit and respect the consistency level so non-SERIAL reads can read Accord's
+            // writes.
             if (transactionalModeWritesThroughAccord)
+            {
                 return tms.migratingAndMigratedRanges.intersects(token);
+            }
 
-            // If we are migrating from a mode that used to write to Accord then any range that isn't migrating/migrated
-            // should continue to write through Accord.
-            // It's not completely symmetrical because Paxos is able to read Accord's writes by performing a single key barrier
-            // and regular mutations will be able to do the same thing (needs to be added along with non-transactional reads)
-            // This means that migrating ranges don't need to be written through Accord because we are running Paxos now
-            // and not Accord. When migrating to Accord we need to do all the writes through Accord even if we aren't
-            // reading through Accord so that repair + Accord metadata is sufficient for Accord to be able to read
-            // safely and deterministically from any coordinator
+            // If we are migrating from a mode that used to write to Accord then any range that isn't migrated
+            // should continue to write through Accord. Accord might still be executing txns pre-migration so continue
+            // to route writes through Accord until migration is completed.
             if (migrationFromWritesThroughAccord)
-                return !tms.migratingAndMigratedRanges.intersects(token);
+                return !tms.migratedRanges.intersects(token);
         }
         return false;
     }

src/java/org/apache/cassandra/service/consensus/migration/ConsensusMigrationState.java

@@ -33,7 +33,6 @@ import javax.annotation.Nullable;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
@@ -147,10 +146,10 @@ public class ConsensusMigrationState implements MetadataValue<ConsensusMigration
 
         Map<Epoch, List<Range<Token>>> migratingRangesByEpoch = ImmutableMap.of();
         if (!ranges.isEmpty())
-            ImmutableMap.of(Epoch.EMPTY, ranges);
+            migratingRangesByEpoch = ImmutableMap.of(Epoch.EMPTY, ranges);
 
         if (overwrite)
-            tableState = new TableMigrationState(metadata.keyspace, metadata.name, metadata.id, target, ImmutableSet.of(), initialRepairPendingRanges(target, ranges), migratingRangesByEpoch);
+            tableState = new TableMigrationState(metadata.keyspace, metadata.name, metadata.id, target, ImmutableList.of(), initialRepairPendingRanges(target, ranges), migratingRangesByEpoch);
         else
             tableState = tableState.withRangesMigrating(ranges, target);
 
@@ -168,6 +167,7 @@ public class ConsensusMigrationState implements MetadataValue<ConsensusMigration
     private static void putUnchanged(Map<TableId, TableMigrationState> current, ImmutableMap.Builder<TableId, TableMigrationState> next, Collection<TableMetadata> changed)
     {
         Set<TableId> changedIds = changed.stream().map(TableMetadata::id).collect(Collectors.toSet());
+
         putUnchanged(current, next, changedIds);
     }
 
@@ -183,12 +183,6 @@ public class ConsensusMigrationState implements MetadataValue<ConsensusMigration
     {
         ImmutableMap.Builder<TableId, TableMigrationState> updated = ImmutableMap.builder();
         putUnchanged(tableStates, updated, new HashSet<>(completed));
-        for (Map.Entry<TableId, TableMigrationState> entry : tableStates.entrySet())
-        {
-            if (completed.contains(entry.getKey()))
-                continue;
-            updated.put(entry);
-        }
         return new ConsensusMigrationState(lastModified, updated.build());
     }
 
@@ -208,7 +202,6 @@ public class ConsensusMigrationState implements MetadataValue<ConsensusMigration
             updated.put(metadata.id, state);
             return new ConsensusMigrationState(lastModified, updated.build());
         }
-
     }
 
     public ConsensusMigrationState withMigrationsRemovedFor(Set<TableId> removed)

src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java

@@ -18,10 +18,15 @@
 
 package org.apache.cassandra.service.consensus.migration;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
+import java.util.function.BiPredicate;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 
 import accord.primitives.Routable.Domain;
 import accord.primitives.Seekables;
@@ -31,25 +36,42 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.NormalizedRanges;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
 import org.apache.cassandra.locator.EndpointsForToken;
 import org.apache.cassandra.locator.ReplicaLayout;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.schema.TableParams;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.consensus.TransactionalMode;
+import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.KeyMigrationState;
 import org.apache.cassandra.service.paxos.Paxos;
+import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.ClusterMetadataService;
-import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static org.apache.cassandra.dht.Range.compareRightToken;
 import static org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.getConsensusMigratedAt;
 import static org.apache.cassandra.service.consensus.migration.ConsensusMigrationTarget.paxos;
 import static org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.ConsensusRoutingDecision.accord;
@@ -59,6 +81,10 @@ import static org.apache.cassandra.service.consensus.migration.ConsensusRequestR
 /**
  * Helper class to decide where to route a request that requires consensus, migrating a key if necessary
  * before rerouting.
+ *
+ * This router has to be used for all SERIAL reads and writes to ensure the correct operation of Paxos/Acocrd during migration
+ * and for all non-SERIAL reads because non-SERIAL reads may end up being routed to Accord and Accord needs CRR to manage
+ * any key migrations that need to be performed
  */
 public class ConsensusRequestRouter
 {
@@ -226,10 +252,18 @@ public class ConsensusRequestRouter
         // Accord -> Paxos - Paxos will ask Accord to migrate in the read at each replica if necessary
         // Paxos -> Accord - Paxos needs to be repaired before Accord runs so do it here
         if (tms.targetProtocol == paxos)
+            // TODO (important): Why are these two cases paxosV2 instead of `pickPaxos`?
+            // Because we only supported PaxosV2 for migration?
+            // Eventually we want to support both so just use pickPaxos and error out on migration from paxosV1 elsewhere?
             return paxosV2;
         else
-            // Should exit exceptionally if the repair is not done
-            ConsensusKeyMigrationState.repairKeyPaxos(naturalReplicas, cm.epoch, key, cfs, consistencyLevel, requestTime, timeoutNanos, isLocallyReplicated, isForWrite);
+        {
+            if (tms.accordSafeToReadRanges.intersects(key.getToken()))
+                // Should exit exceptionally if the repair is not done
+                ConsensusKeyMigrationState.repairKeyPaxos(naturalReplicas, cm.epoch, key, cfs, consistencyLevel, requestTime, timeoutNanos, isLocallyReplicated, isForWrite);
+            else
+                return pickPaxos();
+        }
 
         return pickMigrated(tms.targetProtocol);
     }
@@ -274,31 +308,169 @@ public class ConsensusRequestRouter
         return false;
     }
 
-    public boolean isKeyInMigratingOrMigratedRangeFromAccord(Epoch epoch, TableId tableId, DecoratedKey key)
+    public boolean isRangeManagedByAccordForReadAndWrite(ClusterMetadata cm, TableId tableId, TokenRange range)
     {
-        ClusterMetadata cm = ClusterMetadataService.instance().fetchLogFromCMS(epoch);
+        TableMetadata metadata = getTableMetadata(cm, tableId);
+        TransactionalMode transactionalMode = metadata.params.transactionalMode;
+        TransactionalMigrationFromMode transactionalMigrationFromMode = metadata.params.transactionalMigrationFrom;
         TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId);
-        return isKeyInMigratingOrMigratedRangeFromAccord(getTableMetadata(cm, tableId), tms, key);
+        if (tms == null)
+        {
+            checkState(transactionalMigrationFromMode == TransactionalMigrationFromMode.none, "TableMigrationState shouldn't be null during migration");
+            return transactionalMode.nonSerialReadsThroughAccord;
+        }
+
+        // = token ends up as a min and max key bound in C* parlance and min and max token key in Accord parlance
+        // and the conversion to a C* range results in the unintentional creation of a wrap around range.
+        // Instead treat it like a key and do that check.
+        if (range.start().getClass() == MinTokenKey.class
+            && range.end() instanceof TokenKey
+            && range.start().token().equals(range.end().token()))
+        {
+            checkState(range.end().getClass() != MinTokenKey.class, "Unexpected empty range");
+            return isTokenManagedByAccordForReadAndWrite(metadata, tms, range.start().token());
+        }
+        else if (range.start().getClass() == MinTokenKey.class)
+        {
+            // Start is particularly problematic because we use min MinTokenKey to make start inclusive and this is something
+            // that isn't possible to mimic at all with Range<Token>, for end it's less problematic because just the token
+            // is sufficient for Accord to route the query and select the correct shards even if it might accidentally run
+            // on an extra shard, the filtering will take care of it. There is nothing to do here but convert to a bounds
+            // and use the bounds check
+            PartitionPosition startPP = range.start().token().minKeyBound();
+            PartitionPosition endPP;
+            if (range.end().getClass() == SentinelKey.class)
+                endPP = DatabaseDescriptor.getPartitioner().getMinimumToken().maxKeyBound();
+            else if (range.end().getClass() == MinTokenKey.class)
+                endPP = range.end().token().minKeyBound();
+            else
+                endPP = range.end().token().maxKeyBound();
+            Bounds<PartitionPosition> bounds = new Bounds<>(startPP, endPP);
+            return isBoundsExclusivelyManagedByAccordForRead(transactionalMode, transactionalMigrationFromMode, tms, bounds);
+        }
+        else
+        {
+            return isRangeManagedByAccordForReadAndWrite(metadata,
+                                                         cm.consensusMigrationState.tableStates.get(tableId),
+                                                         range.toKeyspaceRange());
+        }
+    }
+
+    /*
+     * A lightweight check against cluster metadata that doesn't check if the range has already been migrated
+     * using local system table state. It just assumes that the key migration has already been done.
+     *
+     * This version is for is full read write transactions
+     */
+    public boolean isRangeManagedByAccordForReadAndWrite(TableMetadata metadata, TableMigrationState tms, Range<Token> range)
+    {
+        checkState(!range.isTrulyWrapAround(), "Accidentally created a wrap around range");
+        TransactionalMode transactionalMode = metadata.params.transactionalMode;
+        TransactionalMigrationFromMode migrationFrom = metadata.params.transactionalMigrationFrom;
+
+        if (migrationFrom.isMigrating())
+            checkState(tms != null, "Can't have migration in progress without tms");
+
+        if (transactionalMode.accordIsEnabled)
+        {
+            if (!migrationFrom.isMigrating())
+                return true;
+            if (migrationFrom.migratingFromAccord())
+                return true;
+            // Accord can only read/write the key if it is in a safe to read (repaired) range
+            if (Range.intersects(tms.accordSafeToReadRanges, ImmutableList.of(range)))
+                return true;
+        }
+        else
+        {
+            // Once the migration starts only barriers are allowed to run for the key in Accord
+            if (migrationFrom.migratingFromAccord() && !Range.intersects(tms.migratingAndMigratedRanges, ImmutableList.of(range)))
+                return true;
+        }
+
+        return false;
+    }
+
+    public boolean isKeyManagedByAccordForReadAndWrite(ClusterMetadata cm, TableId tableId, DecoratedKey key)
+    {
+        return isTokenManagedByAccordForReadAndWrite(getTableMetadata(cm, tableId),
+                                                   cm.consensusMigrationState.tableStates.get(tableId),
+                                                   key.getToken());
     }
 
     /*
      * A lightweight check against cluster metadata that doesn't check if the key has already been migrated
-     * using local system table state.
+     * using local system table state. It just assumes that the key migration has already been done.
+     *
+     * This version is for is full read write transactions
      */
-    public boolean isKeyInMigratingOrMigratedRangeFromAccord(TableMetadata metadata, TableMigrationState tms, DecoratedKey key)
+    public boolean isTokenManagedByAccordForReadAndWrite(TableMetadata metadata, TableMigrationState tms, Token token)
     {
-        if (!metadata.params.transactionalMigrationFrom.isMigrating())
-            return false;
+        TransactionalMode transactionalMode = metadata.params.transactionalMode;
+        TransactionalMigrationFromMode migrationFrom = metadata.params.transactionalMigrationFrom;
 
-        // No state means no migration for this table
-        if (tms == null)
-            return false;
+        if (migrationFrom.isMigrating())
+            checkState(tms != null, "Can't have migration in progress without tms");
 
-        if (tms.targetProtocol == ConsensusMigrationTarget.accord)
-            return false;
+        if (transactionalMode.accordIsEnabled)
+        {
+            if (!migrationFrom.isMigrating())
+                return true;
+            if (migrationFrom.migratingFromAccord())
+                return true;
+            // Accord can only read/write the key if it is in a safe to read (repaired) range
+            if (tms.accordSafeToReadRanges.intersects(token))
+                return true;
+        }
+        else
+        {
+            // Once the migration starts only barriers are allowed to run for the key in Accord
+            if (migrationFrom.migratingFromAccord() && !tms.migratingAndMigratedRanges.intersects(token))
+                return true;
+        }
 
-        if (tms.migratingAndMigratedRanges.intersects(key.getToken()))
-            return true;
+        return false;
+    }
+
+    public boolean isKeyManagedByAccordForWrite(ClusterMetadata cm, TableId tableId, DecoratedKey key)
+    {
+        return isKeyManagedByAccordForWrite(getTableMetadata(cm, tableId),
+                                            cm.consensusMigrationState.tableStates.get(tableId),
+                                            key);
+    }
+
+    /*
+     * A lightweight check against cluster metadata that doesn't check if the key has already been migrated
+     * using local system table state. It just assumes that the key migration has already been done.
+     *
+     * This version is for writes through Accord before Accord is able to safely read.
+     */
+    public boolean isKeyManagedByAccordForWrite(TableMetadata metadata, TableMigrationState tms, DecoratedKey key)
+    {
+        TransactionalMode transactionalMode = metadata.params.transactionalMode;
+        TransactionalMigrationFromMode migrationFrom = metadata.params.transactionalMigrationFrom;
+        Token token = key.getToken();
+
+        if (migrationFrom.isMigrating())
+            checkState(tms != null, "Can't have migration in progress without tms");
+
+        if (transactionalMode.accordIsEnabled)
+        {
+            if (!migrationFrom.isMigrating())
+                return true;
+            if (migrationFrom.migratingFromAccord())
+                return true;
+            // Accord can blind write to the key even if it isn't safe to read from it so use migratingAndMigratedRanges
+            if (tms.migratingAndMigratedRanges.intersects(token))
+                return true;
+        }
+        else
+        {
+            // We can always allow writes through Accord and it's necessary to do that so that
+            // andy premigration txns aren't exposed to non-transactional writes
+            if (migrationFrom.nonSerialWritesThroughAccord() && !tms.migratedRanges.intersects(token))
+                return true;
+        }
 
         return false;
     }
@@ -340,4 +512,409 @@ public class ConsensusRequestRouter
     {
         return Paxos.useV2() ? paxosV2 : paxosV1;
     }
+
+    public static void validateSafeToReadNonTransactionally(ReadCommand command)
+    {
+        if (command.potentialTxnConflicts().allowed)
+            return;
+
+        String keyspace = command.metadata().keyspace;
+        // System keyspaces are never managed by Accord
+        if (SchemaConstants.isSystemKeyspace(keyspace))
+            return;
+
+        // Local keyspaces are never managed by Accord
+        if (Schema.instance.localKeyspaces().containsKeyspace(keyspace))
+            return;
+
+        ClusterMetadata cm = ClusterMetadata.current();
+        TableId tableId = command.metadata().id;
+        TableMetadata tableMetadata = getTableMetadata(cm, tableId);
+        // Null for local or dropped tables
+        if (tableMetadata == null)
+            return;
+
+        TransactionalMode transactionalMode = tableMetadata.params.transactionalMode;
+        TransactionalMigrationFromMode transactionalMigrationFromMode = tableMetadata.params.transactionalMigrationFrom;
+        if (!transactionalMode.nonSerialReadsThroughAccord && !transactionalMigrationFromMode.nonSerialReadsThroughAccord())
+            return;
+
+        TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tableId);
+
+        // Null with a transaction mode that reads through Accord indicates a completed migration or table created
+        // to use Accord initially
+        if (tms == null)
+        {
+            checkState(transactionalMigrationFromMode == TransactionalMigrationFromMode.none);
+            if (transactionalMode.nonSerialReadsThroughAccord)
+            {
+                ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+                if (cfs != null)
+                    cfs.metric.readsRejectedOnWrongSystem.mark();
+                throw new RetryOnDifferentSystemException();
+            }
+        }
+
+        boolean isExclusivelyReadableFromAccord;
+        if (command.isRangeRequest())
+            isExclusivelyReadableFromAccord = isBoundsExclusivelyManagedByAccordForRead(transactionalMode, transactionalMigrationFromMode, tms, command.dataRange().keyRange());
+        else
+            isExclusivelyReadableFromAccord = isTokenExclusivelyManagedByAccordForRead(transactionalMode, transactionalMigrationFromMode, tms, ((SinglePartitionReadCommand)command).partitionKey().getToken());
+
+        if (isExclusivelyReadableFromAccord)
+        {
+            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId);
+            if (cfs != null)
+                cfs.metric.readsRejectedOnWrongSystem.mark();
+            throw new RetryOnDifferentSystemException();
+        }
+    }
+
+    private static boolean isTokenExclusivelyManagedByAccordForRead(@Nonnull TransactionalMode transactionalMode,
+                                                                     @Nonnull TransactionalMigrationFromMode migrationFrom,
+                                                                     @Nonnull TableMigrationState tms,
+                                                                     @Nonnull Token token)
+    {
+        checkNotNull(transactionalMode, "transactionalMode is null");
+        checkNotNull(migrationFrom, "migrationFrom is null");
+        checkNotNull(tms, "tms (TableMigrationState) is null");
+        checkNotNull(token, "bounds is null");
+
+        if (transactionalMode.accordIsEnabled)
+        {
+            if (!migrationFrom.isMigrating())
+                return true;
+            if (migrationFrom.migratingFromAccord())
+                return true;
+
+            // Accord is exclusive once the range is fully migrated to Accord, but possible to read from safely
+            // when accordSafeToReadRanges covers the entire bound
+            if (tms.migratedRanges.intersects(token))
+                return true;
+        }
+        else
+        {
+            // Once the migration starts only barriers are allowed to run for the key in Accord
+            if (migrationFrom.migratingFromAccord() && !tms.migratingAndMigratedRanges.intersects(token))
+                return true;
+        }
+
+        return false;
+    }
+
+    // Returns true if any part of the bound
+    private static boolean isBoundsExclusivelyManagedByAccordForRead(@Nonnull TransactionalMode transactionalMode,
+                                                                     @Nonnull TransactionalMigrationFromMode migrationFrom,
+                                                                     @Nonnull TableMigrationState tms,
+                                                                     @Nonnull AbstractBounds<PartitionPosition> bounds)
+    {
+        checkNotNull(transactionalMode, "transactionalMode is null");
+        checkNotNull(migrationFrom, "migrationFrom is null");
+        checkNotNull(tms, "tms (TableMigrationState) is null");
+        checkNotNull(bounds, "bounds is null");
+
+        BiPredicate<AbstractBounds<PartitionPosition>, NormalizedRanges<Token>> intersects = (testBounds, testRanges) -> {
+            // TODO (nicetohave): Efficiency of this intersection
+            for (org.apache.cassandra.dht.Range<Token> range : testRanges)
+            {
+                Pair<AbstractBounds<PartitionPosition>, AbstractBounds<PartitionPosition>> intersectionAndRemainder = Range.intersectionAndRemainder(testBounds, range);
+                return intersectionAndRemainder.left != null;
+            }
+            return false;
+        };
+
+        if (bounds.left.getToken().equals(bounds.right.getToken()) && !bounds.inclusiveLeft() && bounds.inclusiveRight())
+        {
+            return isTokenExclusivelyManagedByAccordForRead(transactionalMode, migrationFrom, tms, bounds.left.getToken());
+        }
+
+        if (transactionalMode.accordIsEnabled)
+        {
+            if (!migrationFrom.isMigrating())
+                return true;
+            if (migrationFrom.migratingFromAccord())
+                return true;
+
+            // Accord is exclusive once the range is fully migrated to Accord, but possible to read from safely
+            // when accordSafeToReadRanges covers the entire bound
+            if (intersects.test(bounds, tms.migratedRanges))
+                return true;
+        }
+        else
+        {
+            // Once the migration starts only barriers are allowed to run for the key in Accord
+            if (migrationFrom.migratingFromAccord() && !intersects.test(bounds, tms.migratingAndMigratedRanges))
+                return true;
+        }
+
+        return false;
+    }
+
+    public enum RangeReadTarget
+    {
+        accord,
+        normal
+    }
+
+    public static class RangeReadWithTarget
+    {
+        public final PartitionRangeReadCommand read;
+        public final RangeReadTarget target;
+
+        private RangeReadWithTarget(PartitionRangeReadCommand read, RangeReadTarget target)
+        {
+            this.read = read;
+            this.target = target;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "RangeReadWithTarget{" +
+                   "read=" + read +
+                   ", target=" + target +
+                   '}';
+        }
+    }
+
+    /**
+     * While it's possible to map the Accord read to a single txn it doesn't seem worth it since it's a pretty unusual
+     * scenario where we do this during migration and have a lot of different read commands.
+     */
+    public static List<RangeReadWithTarget> splitReadIntoAccordAndNormal(ClusterMetadata cm, PartitionRangeReadCommand read, ReadCoordinator readCoordinator, Dispatcher.RequestTime requestTime)
+    {
+        if (!readCoordinator.isEventuallyConsistent())
+            return ImmutableList.of(new RangeReadWithTarget(read, RangeReadTarget.normal));
+        TableMetadata tm = getTableMetadata(cm, read.metadata().id);
+        if (tm == null || (!tm.params.transactionalMode.nonSerialReadsThroughAccord && !tm.params.transactionalMigrationFrom.nonSerialReadsThroughAccord()))
+            return ImmutableList.of(new RangeReadWithTarget(read, RangeReadTarget.normal));
+
+        List<RangeReadWithTarget> result = null;
+        TransactionalMode transactionalMode = tm.params.transactionalMode;
+        TransactionalMigrationFromMode transactionalMigrationFromMode = tm.params.transactionalMigrationFrom;
+        boolean transactionalModeReadsThroughAccord = transactionalMode.nonSerialReadsThroughAccord;
+        RangeReadTarget migrationToTarget = transactionalModeReadsThroughAccord ? RangeReadTarget.accord : RangeReadTarget.normal;
+        boolean migrationFromReadsThroughAccord = transactionalMigrationFromMode.nonSerialReadsThroughAccord();
+        RangeReadTarget migrationFromTarget = migrationFromReadsThroughAccord ? RangeReadTarget.accord : RangeReadTarget.normal;
+        TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id);
+        if (tms == null)
+        {
+            if (transactionalMigrationFromMode == TransactionalMigrationFromMode.none)
+                // There is no migration and no TMS so do what the schema says since no migration should be required
+                return ImmutableList.of(new RangeReadWithTarget(read, transactionalModeReadsThroughAccord ? RangeReadTarget.accord : RangeReadTarget.normal));
+            else
+                // If we are migrating from something and there is no migration state the migration hasn't begun
+                // so continue to do what we are migrating from does until the range is marked as migrating
+                return ImmutableList.of(new RangeReadWithTarget(read, migrationFromReadsThroughAccord ? RangeReadTarget.accord : RangeReadTarget.normal));
+        }
+
+
+        // AbstractBounds can potentially be left/right inclusive while Range used to track migration is only right inclusive
+        // The right way to tackle this seems to be to find the tokens that intersect the key range and then split until
+        // until nothing intersects
+        AbstractBounds<PartitionPosition> keyRange = read.dataRange().keyRange();
+        AbstractBounds<PartitionPosition> remainder = keyRange;
+
+        // Migrating to Accord we only read through Accord when the range is fully migrated, but migrating back
+        // we stop reading from Accord as soon as the range is marked migrating and do key migration on read
+        NormalizedRanges<Token> migratedRanges = transactionalModeReadsThroughAccord ? tms.migratedRanges : tms.migratingAndMigratedRanges;
+
+        // Add the preceding range if any
+        if (!migratedRanges.isEmpty())
+        {
+            Token firstMigratingToken = migratedRanges.get(0).left.getToken();
+            int leftCmp = keyRange.left.getToken().compareTo(firstMigratingToken);
+            int rightCmp = compareRightToken(keyRange.right.getToken(), firstMigratingToken);
+            if (leftCmp <= 0)
+            {
+                if (rightCmp <= 0)
+                    return ImmutableList.of(new RangeReadWithTarget(read, migrationFromTarget));
+                result = new ArrayList<>();
+                AbstractBounds<PartitionPosition> precedingRange = keyRange.withNewRight(rightCmp <= 0 ? keyRange.right : firstMigratingToken.maxKeyBound());
+                // Could be an empty bound, it's fine to let a min KeyBound and max KeyBound through as that isn't empty
+                if (!precedingRange.left.equals(precedingRange.right))
+                    result.add(new RangeReadWithTarget(read.forSubRange(precedingRange, false), migrationFromTarget));
+            }
+        }
+
+        boolean hadAccordReads = false;
+        for (Range<Token> r : migratedRanges)
+        {
+            Pair<AbstractBounds<PartitionPosition>, AbstractBounds<PartitionPosition>> intersectionAndRemainder = Range.intersectionAndRemainder(remainder, r);
+            if (intersectionAndRemainder.left != null)
+            {
+                if (result == null)
+                    result = new ArrayList<>();
+                PartitionRangeReadCommand subRead = read.forSubRange(intersectionAndRemainder.left, result.isEmpty() ? true : false);
+                result.add(new RangeReadWithTarget(subRead, migrationToTarget));
+                hadAccordReads = true;
+            }
+            remainder = intersectionAndRemainder.right;
+            if (remainder == null)
+                break;
+        }
+
+        if (remainder != null)
+        {
+            if (result != null)
+                result.add(new RangeReadWithTarget(read.forSubRange(remainder, true), migrationFromTarget));
+            else
+                return ImmutableList.of(new RangeReadWithTarget(read.forSubRange(remainder, false), migrationFromTarget));
+        }
+
+        checkState(result != null && !result.isEmpty(), "Shouldn't have null or empty result");
+        checkState(result.get(0).read.dataRange().startKey().equals(read.dataRange().startKey()), "Split reads should encompass entire range");
+        checkState(result.get(result.size() - 1).read.dataRange().stopKey().equals(read.dataRange().stopKey()), "Split reads should encompass entire range");
+        if (result.size() > 1)
+        {
+            for (int i = 0; i < result.size() - 1; i++)
+            {
+                checkState(result.get(i).read.dataRange().stopKey().equals(result.get(i + 1).read.dataRange().startKey()), "Split reads should all be adjacent");
+                checkState(result.get(i).target != result.get(i + 1).target, "Split reads should be for different targets");
+            }
+        }
+
+        //TODO (later): https://issues.apache.org/jira/browse/CASSANDRA-20211 Range reads could use a barrier
+        if (hadAccordReads)
+        {
+            // do barrier
+        }
+
+        return result;
+    }
+
+    /**
+     * Result of splitting mutations across Accord and non-transactional boundaries
+     */
+    public static class SplitReads
+    {
+        @Nullable
+        public final SinglePartitionReadCommand.Group accordReads;
+
+        @Nullable
+        public final SinglePartitionReadCommand.Group normalReads;
+
+        private SplitReads(SinglePartitionReadCommand.Group accordReads, SinglePartitionReadCommand.Group normalReads)
+        {
+            this.accordReads = accordReads;
+            this.normalReads = normalReads;
+        }
+    }
+
+    public static SplitReads splitReadsIntoAccordAndNormal(ClusterMetadata cm, SinglePartitionReadCommand.Group reads, ReadCoordinator coordinator, Dispatcher.RequestTime requestTime)
+    {
+        if (!coordinator.isEventuallyConsistent())
+            return new SplitReads(null, reads);
+        List<SinglePartitionReadCommand> accordReads = null;
+        List<SinglePartitionReadCommand> normalReads = null;
+
+        TableMetadata tm = getTableMetadata(cm, reads.queries.get(0).metadata().id);
+        if (tm == null || (!tm.params.transactionalMode.nonSerialReadsThroughAccord && !tm.params.transactionalMigrationFrom.nonSerialReadsThroughAccord()))
+            return new SplitReads(null, reads);
+
+        TransactionalMode transactionalMode = tm.params.transactionalMode;
+        TransactionalMigrationFromMode transactionalMigrationFromMode = tm.params.transactionalMigrationFrom;
+        TableMigrationState tms = cm.consensusMigrationState.tableStates.get(tm.id);
+
+        for (SinglePartitionReadCommand command : reads.queries)
+        {
+            if (tokenShouldBeReadThroughAccord(tms, command.partitionKey().getToken(), transactionalMode, transactionalMigrationFromMode))
+            {
+                if (accordReads == null)
+                    accordReads = new ArrayList<>(reads.queries.size());
+                accordReads.add(command);
+            }
+            else
+            {
+                if (normalReads == null)
+                    normalReads = new ArrayList<>(reads.queries.size());
+                normalReads.add(command);
+            }
+        }
+
+        // When migrating from Accord -> Paxos we need to do the Accord barrier to have acknowledged Accord writes
+        // be visible to non-SERIAL reads, but from Paxos -> Accord we don't need to because read only transactions
+        // don't have recovery determinism issues and Accord will honor read consistency levels and match the behavior
+        // of non-serially reading Paxos transactions. Since it's a non-SERIAL read there is no guarantee of seeing
+        // in-flight Paxos operations, for that you would need to read at SERIAL.
+        // If the migration direction is from a mode that used to read through Accord then Accord would be
+        // doing async commit so we need barriers if this mode is no longer reading through Accord.
+        if (transactionalMigrationFromMode.isMigrating() && transactionalMigrationFromMode.nonSerialReadsThroughAccord() && !transactionalMode.nonSerialReadsThroughAccord && normalReads != null)
+        {
+            checkState(!normalReads.isEmpty());
+            List<DecoratedKey> keysNeedingBarrier = null;
+            long maxRequiredEpoch = Long.MIN_VALUE;
+            for (SinglePartitionReadCommand readCommand : normalReads)
+            {
+                DecoratedKey key = readCommand.partitionKey();
+                KeyMigrationState kms = ConsensusKeyMigrationState.getKeyMigrationState(cm, tms, key);
+                if (!kms.paxosReadSatisfiedByKeyMigration())
+                {
+                    if (keysNeedingBarrier == null)
+                        keysNeedingBarrier = new ArrayList<>(normalReads.size());
+                    keysNeedingBarrier.add(key);
+                    maxRequiredEpoch = Math.max(tms.minMigrationEpoch(key.getToken()).getEpoch(), maxRequiredEpoch);
+                }
+            }
+
+            if (keysNeedingBarrier != null)
+            {
+                checkState(!keysNeedingBarrier.isEmpty());
+                checkState(maxRequiredEpoch != Long.MIN_VALUE);
+                // Local barriers don't support multiple keys so create a global one unless there is a single key
+                // See BarrierType enum for explanation of global vs local
+                boolean global = keysNeedingBarrier.size() > 1 ? true : false;
+                ConsensusKeyMigrationState.repairKeysAccord(keysNeedingBarrier, tm.id, maxRequiredEpoch, requestTime, global, false);
+            }
+        }
+
+        SinglePartitionReadCommand.Group accordGroup = accordReads != null ? SinglePartitionReadCommand.Group.create(accordReads, reads.limits()) : null;
+        SinglePartitionReadCommand.Group normalGroup = normalReads != null ? SinglePartitionReadCommand.Group.create(normalReads, reads.limits()) : null;
+        return new SplitReads(accordGroup, normalGroup);
+    }
+
+    private static boolean tokenShouldBeReadThroughAccord(TableMigrationState tms,
+                                                          @Nonnull Token token,
+                                                          @Nonnull TransactionalMode transactionalMode,
+                                                          TransactionalMigrationFromMode transactionalMigrationFromMode)
+    {
+        boolean transactionalModeReadsThroughAccord = transactionalMode.nonSerialReadsThroughAccord;
+        boolean migrationFromReadsThroughAccord = transactionalMigrationFromMode.nonSerialReadsThroughAccord();
+
+        if (transactionalModeReadsThroughAccord && migrationFromReadsThroughAccord)
+            return true;
+
+        // Could be migrating or could be completely migrated, if it's migrating check if the key for this mutation
+        if (transactionalModeReadsThroughAccord || migrationFromReadsThroughAccord)
+        {
+            if (tms == null)
+            {
+                if (transactionalMigrationFromMode == TransactionalMigrationFromMode.none)
+                    // There is no migration and no TMS so do what the schema says since no migration should be required
+                    return transactionalModeReadsThroughAccord;
+                else
+                    // If we are migrating from something and there is no migration state the migration hasn't begun
+                    // so continue to do what we are migrating from does until the range is marked as migrating
+                    return migrationFromReadsThroughAccord;
+            }
+
+            // In theory we can start reading from Accord immediately because we know these transactions are 100%
+            // read only but then that impacts performance more so wait for the range to be completely migrated
+            // when it can potentially do single replica reads
+            if (transactionalModeReadsThroughAccord)
+                return tms.migratedRanges.intersects(token);
+
+            // If we are migrating from a mode that used to write to Accord then any range that isn't migrating/migrated
+            // should continue to write through Accord.
+            // It's not completely symmetrical because Paxos is able to read Accord's writes by performing a single key barrier
+            // and regular mutations will be able to do the same thing (needs to be added along with non-transactional reads)
+            // This means that migrating ranges don't need to be written through Accord because we are running Paxos now
+            // and not Accord. When migrating to Accord we need to do all the writes through Accord even if we aren't
+            // reading through Accord so that repair + Accord metadata is sufficient for Accord to be able to read
+            // safely and deterministically from any coordinator
+            if (migrationFromReadsThroughAccord)
+                return !tms.migratingAndMigratedRanges.intersects(token);
+        }
+        return false;
+    }
 }

src/java/org/apache/cassandra/service/consensus/migration/ConsensusTableMigration.java

@@ -52,7 +52,6 @@ import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.accord.TokenRange;
-import org.apache.cassandra.service.consensus.TransactionalMode;
 import org.apache.cassandra.service.paxos.Paxos;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.tcm.ClusterMetadata;
@@ -171,31 +170,26 @@ public abstract class ConsensusTableMigration
         return cm.consensusMigrationState.tableStates.get(tableId);
     }
 
-    public static void startMigrationToConsensusProtocol(@Nonnull String targetProtocolName,
-                                                         @Nullable List<String> keyspaceNames,
+    public static void startMigrationToConsensusProtocol(@Nullable List<String> keyspaceNames,
                                                          @Nonnull Optional<List<String>> maybeTables,
                                                          @Nonnull Optional<String> maybeRangesStr)
     {
         checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty");
-        ConsensusMigrationTarget targetProtocol = ConsensusMigrationTarget.fromString(targetProtocolName);
+        ClusterMetadata cm = ClusterMetadata.current();
 
         if (keyspaceNames == null || keyspaceNames.isEmpty())
         {
             keyspaceNames = ImmutableList.copyOf(StorageService.instance.getNonLocalStrategyKeyspaces());
         }
         checkState(keyspaceNames.size() == 1 || !maybeTables.isPresent(), "Can't specify tables with multiple keyspaces");
-        List<TableId> ids = keyspacesAndTablesToTableIds(keyspaceNames, maybeTables);
+        List<TableId> ids = keyspacesAndTablesToTableIds(cm, keyspaceNames, maybeTables);
 
-        // TODO (review): should this perform the schema change to make these tables accord tables?
         List<TableId> tableIds = new ArrayList<>();
         for (TableId tableId : ids)
         {
-            TableMetadata metadata = Schema.instance.getTableMetadata(tableId);
+            TableMetadata metadata = cm.schema.getTableMetadata(tableId);
             if (metadata == null || !metadata.params.transactionalMigrationFrom.isMigrating())
                 continue;
-            TransactionalMode transactionalMode = metadata.params.transactionalMode;
-            if (!transactionalMode.nonSerialWritesThroughAccord && transactionalMode != TransactionalMode.unsafe_writes)
-                throw new IllegalStateException("non-SERIAL writes need to be routed through Accord before attempting migration, or enable mixed mode");
             tableIds.add(tableId);
         }
 
@@ -207,41 +201,41 @@ public abstract class ConsensusTableMigration
         Token minToken = partitioner.getMinimumToken();
         NormalizedRanges<Token> ranges = normalizedRanges(maybeParsedRanges.orElse(ImmutableList.of(new Range(minToken, minToken))));
 
-
-        ClusterMetadataService.instance().commit(new BeginConsensusMigrationForTableAndRange(targetProtocol, ranges, tableIds));
+        ClusterMetadataService.instance().commit(new BeginConsensusMigrationForTableAndRange(ranges, tableIds));
     }
 
     public static Integer finishMigrationToConsensusProtocol(@Nonnull String keyspace,
-                                                                   @Nonnull Optional<List<String>> maybeTables,
-                                                                   @Nonnull Optional<String> maybeRangesStr,
-                                                                   ConsensusMigrationTarget target)
+                                                             @Nonnull Optional<List<String>> maybeTables,
+                                                             @Nonnull Optional<String> maybeRangesStr,
+                                                             @Nonnull ConsensusMigrationTarget target)
     {
         checkArgument(!maybeTables.isPresent() || !maybeTables.get().isEmpty(), "Must provide at least 1 table if Optional is not empty");
         checkNotNull(target);
+        ClusterMetadata cm = ClusterMetadata.current();
 
         Optional<List<Range<Token>>> localKeyspaceRanges = Optional.of(ImmutableList.copyOf(StorageService.instance.getLocalReplicas(keyspace).onlyFull().ranges()));
         List<Range<Token>> ranges = maybeRangesToRanges(maybeRangesStr, localKeyspaceRanges);
         Map<TableId, TableMigrationState> allTableMigrationStates = ClusterMetadata.current().consensusMigrationState.tableStates;
-        List<TableId> tableIds = keyspacesAndTablesToTableIds(ImmutableList.of(keyspace), maybeTables, Optional.of(allTableMigrationStates::containsKey));
+        List<TableId> tableIds = keyspacesAndTablesToTableIds(cm, ImmutableList.of(keyspace), maybeTables, Optional.of(allTableMigrationStates::containsKey));
 
         checkState(tableIds.stream().allMatch(allTableMigrationStates::containsKey), "All tables need to be migrating");
         List<TableMigrationState> tableMigrationStates = new ArrayList<>();
         tableIds.forEach(table -> {
-            ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(table);
-            if (cfs == null)
+            TableMetadata tm = cm.schema.getTableMetadata(table);
+            if (tm == null)
             {
-                logger.warn("Table {} does not exist or was dropped", cfs);
+                logger.warn("Table {} does not exist or was dropped", table);
                 return;
             }
             TableMigrationState tms = allTableMigrationStates.get(table);
             if (tms == null)
             {
-                logger.warn("Table {} does not have any migration state", cfs.name);
+                logger.warn("Table {} does not have any migration state", tm.name);
                 return;
             }
-            if(!Range.intersects(ranges, tms.migratingRanges))
+            if (!Range.intersects(ranges, tms.migratingRanges))
             {
-                logger.warn("Table {} with migrating ranges {} does not intersect with any requested ranges {}", cfs.name, tms.migratingRanges, ranges);
+                logger.warn("Table {} with migrating ranges {} does not intersect with any requested ranges {}", tm.name, tms.migratingRanges, ranges);
                 return;
             }
             tableMigrationStates.add(tms);
@@ -341,12 +335,12 @@ public abstract class ConsensusTableMigration
     }
 
 
-    private static List<TableId> keyspacesAndTablesToTableIds(@Nonnull List<String> keyspaceNames, @Nonnull Optional<List<String>> maybeTables)
+    private static List<TableId> keyspacesAndTablesToTableIds(@Nonnull ClusterMetadata cm, @Nonnull List<String> keyspaceNames, @Nonnull Optional<List<String>> maybeTables)
     {
-        return keyspacesAndTablesToTableIds(keyspaceNames, maybeTables, Optional.empty());
+        return keyspacesAndTablesToTableIds(cm, keyspaceNames, maybeTables, Optional.empty());
     }
 
-    private static List<TableId> keyspacesAndTablesToTableIds(@Nonnull List<String> keyspaceNames, @Nonnull Optional<List<String>> maybeTables, @Nonnull Optional<Predicate<TableId>> includeTable)
+    private static List<TableId> keyspacesAndTablesToTableIds(@Nonnull ClusterMetadata cm, @Nonnull List<String> keyspaceNames, @Nonnull Optional<List<String>> maybeTables, @Nonnull Optional<Predicate<TableId>> includeTable)
     {
         List<TableId> tableIds = new ArrayList<>();
         for (String keyspaceName : keyspaceNames)
@@ -355,7 +349,7 @@ public abstract class ConsensusTableMigration
                     tableNames
                             .stream()
                             .map(tableName -> {
-                                TableMetadata tm = Schema.instance.getTableMetadata(keyspaceName, tableName);
+                                TableMetadata tm = cm.schema.getTableMetadata(keyspaceName, tableName);
                                 if (tm == null)
                                     throw new IllegalArgumentException(format("Unknown table %s.%s", keyspaceName, tableName));
                                 return tm.id;
@@ -363,7 +357,7 @@ public abstract class ConsensusTableMigration
                             .collect(toImmutableList()));
             tableIds.addAll(
                     maybeTableIds.orElseGet(() ->
-                            Schema.instance.getKeyspaceInstance(keyspaceName).getColumnFamilyStores()
+                            cm.schema.getKeyspace(keyspaceName).getColumnFamilyStores()
                                     .stream()
                                     .map(ColumnFamilyStore::getTableId)
                                     .filter(includeTable.orElse(Predicates.alwaysTrue())) // Filter out non-migrating so they don't generate an error

src/java/org/apache/cassandra/service/consensus/migration/TableMigrationState.java

@@ -37,12 +37,14 @@ import javax.annotation.Nullable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.NormalizedRanges;
 import org.apache.cassandra.dht.Range;
@@ -58,8 +60,8 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static org.apache.cassandra.db.TypeSizes.sizeof;
-import static org.apache.cassandra.dht.Range.normalize;
 import static org.apache.cassandra.dht.NormalizedRanges.normalizedRanges;
+import static org.apache.cassandra.dht.Range.normalize;
 import static org.apache.cassandra.dht.Range.subtract;
 import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap;
 import static org.apache.cassandra.utils.CollectionSerializers.deserializeSet;
@@ -119,11 +121,19 @@ public class TableMigrationState
     public final NormalizedRanges<Token> repairPendingRanges;
 
     /**
-     * Ranges that are migrating could be in either phase when migrating to Accord. PAxos only has one phase.
+     * Ranges that are migrating could be in either phase when migrating to Accord. Paxos only has one phase.
      */
     @Nonnull
     public final NormalizedRanges<Token> migratingAndMigratedRanges;
 
+    /**
+     * Same as migratingAndMigratedRanges if migrating to Paxos, otherwise migratingAndMigratedRanges.subtract(repairPendingRanges)
+     *
+     * Not included in equals or hashCode because it is inferred from other fields
+     */
+    @Nonnull
+    public final NormalizedRanges<Token> accordSafeToReadRanges;
+
     public TableMigrationState(@Nonnull String keyspaceName,
                                @Nonnull String tableName,
                                @Nonnull TableId tableId,
@@ -145,6 +155,7 @@ public class TableMigrationState
                               .collect(Collectors.toList()));
         this.migratingRanges = normalizedRanges(migratingRangesByEpoch.values().stream().flatMap(Collection::stream).collect(Collectors.toList()));
         this.migratingAndMigratedRanges = normalizedRanges(ImmutableList.<Range<Token>>builder().addAll(migratedRanges).addAll(migratingRanges).build());
+        this.accordSafeToReadRanges = !repairPendingRanges.isEmpty() ? migratingAndMigratedRanges.subtract(this.repairPendingRanges) : migratingAndMigratedRanges;
     }
 
     static List<Range<Token>> initialRepairPendingRanges(ConsensusMigrationTarget target, List<Range<Token>> initialMigratingRanges)
@@ -415,6 +426,11 @@ public class TableMigrationState
         }
     };
 
+    public Iterable<Range<PartitionPosition>> migratedRangesAsPartitionPosition()
+    {
+        return Iterables.transform(migratedRanges, range -> new Range<>(range.left.maxKeyBound(), range.right.maxKeyBound()));
+    }
+
     @Override
     public String toString()
     {

src/java/org/apache/cassandra/service/consensus/migration/TransactionalMigrationFromMode.java

@@ -30,10 +30,10 @@ public enum TransactionalMigrationFromMode
 {
     none(null),  // No migration is in progress. The currently active transaction system could be either Accord or Paxos.
     off(TransactionalMode.off),
-    unsafe(TransactionalMode.unsafe),
-    unsafe_writes(TransactionalMode.unsafe_writes),
     mixed_reads(TransactionalMode.mixed_reads),
     full(TransactionalMode.full),
+    test_unsafe(TransactionalMode.test_unsafe),
+    test_unsafe_writes(TransactionalMode.test_unsafe_writes),
     test_interop_read(TransactionalMode.test_interop_read);
 
     public final TransactionalMode from;
@@ -52,11 +52,11 @@ public enum TransactionalMigrationFromMode
         {
             default: throw new IllegalArgumentException();
             case off: return off;
-            case unsafe: return unsafe;
-            case unsafe_writes: return unsafe_writes;
             case mixed_reads: return mixed_reads;
             case full: return full;
             case test_interop_read: return test_interop_read;
+            case test_unsafe: return test_unsafe;
+            case test_unsafe_writes: return test_unsafe_writes;
         }
     }
 

src/java/org/apache/cassandra/service/paxos/Commit.java

@@ -29,6 +29,7 @@ import com.google.common.base.Objects;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.DeserializationHelper;
 import org.apache.cassandra.io.IVersionedSerializer;
@@ -314,7 +315,7 @@ public class Commit
 
     public Mutation makeMutation()
     {
-        return new Mutation(update, true);
+        return new Mutation(update, PotentialTxnConflicts.ALLOW);
     }
 
     @Override

src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java

@@ -138,7 +138,7 @@ public abstract class AbstractReadExecutor
     {
         boolean hasLocalEndpoint = false;
         Message<ReadCommand> message = null;
-        readCommand = coordinator.maybeAllowOutOfRangeReads(readCommand);
+        readCommand = coordinator.maybeAllowOutOfRangeReads(readCommand, replicaPlan().consistencyLevel());
 
         for (Replica replica: replicas)
         {

src/java/org/apache/cassandra/service/reads/ReadCallback.java

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
+import com.google.common.collect.ImmutableMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,10 +33,12 @@ import org.apache.cassandra.db.MessageParams;
 import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadResponse;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
 import org.apache.cassandra.exceptions.ReadFailureException;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.RequestFailure;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.ReplicaPlan;
@@ -54,6 +57,8 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater;
+import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND;
+import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM;
 import static org.apache.cassandra.tracing.Tracing.isTracing;
 import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
 
@@ -173,6 +178,34 @@ public class ReadCallback<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
         if (snapshot != null)
             snapshot.maybeAbort(command, replicaPlan().consistencyLevel(), received, replicaPlan().readQuorum(), resolver.isDataPresent(), failureReasonByEndpoint);
 
+        // failures keeps incrementing, and this.failureReasonByEndpoint keeps getting new entries after signaling.
+        // Simpler to reason about what happened by copying this.failureReasonByEndpoint and then inferring
+        // failures from it
+        final Map<InetAddressAndPort, RequestFailureReason> failureReasonByEndpoint = ImmutableMap.copyOf(this.failureReasonByEndpoint);
+        int transactionRetryErrors = 0;
+        int coordinatorBehindErrors = 0;
+        for (RequestFailureReason reason : failureReasonByEndpoint.values())
+        {
+            if (reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM)
+                transactionRetryErrors++;
+            if (reason == COORDINATOR_BEHIND)
+                coordinatorBehindErrors++;
+        }
+        int totalRetriableFailures = transactionRetryErrors + coordinatorBehindErrors;
+
+        // TODO (nicetohave): This could be smarter and check if retrying would succeed instead of pessimistically
+        // failing unless all errors are retriable
+        if (!timedout && totalRetriableFailures > 0 && totalRetriableFailures == failureReasonByEndpoint.size())
+        {
+            // Doesn't matter which we throw really but for clarity/metrics be specific
+            // Retrying on the correct system might make this write succeed
+            if (transactionRetryErrors > 0)
+                throw new RetryOnDifferentSystemException();
+            if (coordinatorBehindErrors > 0)
+                throw new CoordinatorBehindException("Read request failed due to coordinator behind");
+        }
+
+
         // Same as for writes, see AbstractWriteResponseHandler
         throw !timedout
             ? new ReadFailureException(replicaPlan().consistencyLevel(), received, replicaPlan().readQuorum(), resolver.isDataPresent(), failureReasonByEndpoint)

src/java/org/apache/cassandra/service/reads/ReadCoordinator.java

@@ -18,8 +18,10 @@
 
 package org.apache.cassandra.service.reads;
 
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts;
 import org.apache.cassandra.db.ReadResponse;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.EndpointsForToken;
@@ -64,16 +66,16 @@ public interface ReadCoordinator
 
     boolean localReadSupported();
     EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, TableId tableId, Token token);
-    default ReadCommand maybeAllowOutOfRangeReads(ReadCommand command)
+    default ReadCommand maybeAllowOutOfRangeReads(ReadCommand command, ConsistencyLevel cl)
     {
         return command;
     }
     void sendReadCommand(Message<ReadCommand> message, InetAddressAndPort to, RequestCallback<ReadResponse> callback);
     default void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests) {}
     void sendReadRepairMutation(Message<Mutation> message, InetAddressAndPort to, RequestCallback<Object> callback);
-    default boolean allowsPotentialTransactionConflicts()
+    default PotentialTxnConflicts potentialTxnConflicts()
     {
-        return !isEventuallyConsistent();
+        return isEventuallyConsistent() ? PotentialTxnConflicts.DISALLOW : PotentialTxnConflicts.ALLOW;
     }
     boolean isEventuallyConsistent();
 }

src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java

@@ -156,7 +156,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
 
         ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 
-        if (source.isSelf())
+        if (source.isSelf() && coordinator.localReadSupported())
         {
             Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler, requestTime));
         }
@@ -164,6 +164,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
         {
             if (source.isTransient())
                 cmd = cmd.copyAsTransientQuery(source);
+            cmd = coordinator.maybeAllowOutOfRangeReads(cmd, consistency);
             MessagingService.instance().sendWithCallback(cmd.createMessage(false, requestTime), source.endpoint(), handler);
         }
 

src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java

@@ -182,7 +182,7 @@ public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowI
     private <E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
     UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared<E, P> replicaPlan)
     {
-        cmd = coordinator.maybeAllowOutOfRangeReads(cmd);
+        cmd = coordinator.maybeAllowOutOfRangeReads(cmd, replicaPlan.get().consistencyLevel());
         DataResolver<E, P> resolver = new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair<E, P>)NoopReadRepair.instance, requestTime);
         ReadCallback<E, P> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
 

src/java/org/apache/cassandra/service/reads/range/AccordRangeResponse.java

@@ -23,15 +23,14 @@ import java.util.function.IntPredicate;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.StorageProxy.ConsensusAttemptResult;
 import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.AbstractIterator;
 
-import static com.google.common.base.Preconditions.checkState;
-
-public class AccordRangeResponse extends AbstractIterator<RowIterator> implements IRangeResponse
+public class AccordRangeResponse extends AbstractIterator<RowIterator> implements PartitionIterator
 {
     private final AsyncTxnResult asyncTxnResult;
     // Range queries don't support reverse, but dutifully threading it through anyways
@@ -55,8 +54,9 @@ public class AccordRangeResponse extends AbstractIterator<RowIterator> implement
         IntPredicate alwaysTrue = ignored -> true;
         IntPredicate alwaysFalse = ignored -> false;
         // TODO (required): Handle retry on different system
-        ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, 1, reversed ? alwaysTrue : alwaysFalse, cl, requestTime);
-        checkState(!consensusAttemptResult.shouldRetryOnNewConsensusProtocol, "Live migration is not supported yet");
+        ConsensusAttemptResult consensusAttemptResult = StorageProxy.getConsensusAttemptResultFromAsyncTxnResult(asyncTxnResult, 1, reversed ? alwaysTrue : alwaysFalse);
+        if (consensusAttemptResult.shouldRetryOnNewConsensusProtocol)
+            throw new RetryOnDifferentSystemException();
         result = consensusAttemptResult.serialReadResult;
     }
 

src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java

@@ -21,37 +21,40 @@ package org.apache.cassandra.service.reads.range;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.PartitionRangeReadCommand;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.filter.DataLimits;
 import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.PartitionIterators;
 import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.exceptions.CoordinatorBehindException;
 import org.apache.cassandra.exceptions.ReadAbortException;
 import org.apache.cassandra.exceptions.ReadFailureException;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.RetryOnDifferentSystemException;
 import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.locator.EndpointsForRange;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.metrics.ClientRangeRequestMetrics;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.service.StorageProxy;
 import org.apache.cassandra.service.accord.IAccordService.AsyncTxnResult;
-import org.apache.cassandra.service.consensus.TransactionalMode;
-import org.apache.cassandra.service.consensus.migration.TransactionalMigrationFromMode;
+import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
+import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadTarget;
+import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadWithTarget;
 import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.ReadCoordinator;
@@ -62,6 +65,9 @@ import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.AbstractIterator;
 import org.apache.cassandra.utils.CloseableIterator;
 
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
+import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 @VisibleForTesting
@@ -187,6 +193,47 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
         return concurrencyFactor;
     }
 
+    private PartitionIterator executeAccord(ClusterMetadata cm, PartitionRangeReadCommand rangeCommand, ConsistencyLevel cl)
+    {
+        //TODO (nicetohave): https://issues.apache.org/jira/browse/CASSANDRA-20210 More efficient reads across command stores
+        AsyncTxnResult result = StorageProxy.readWithAccord(cm, rangeCommand, rangeCommand.dataRange().keyRange(), cl, requestTime);
+        return new AccordRangeResponse(result, rangeCommand.isReversed(), cl, requestTime);
+    }
+
+    private SingleRangeResponse executeNormal(ReplicaPlan.ForRangeRead replicaPlan, PartitionRangeReadCommand rangeCommand, ReadCoordinator readCoordinator)
+    {
+        rangeCommand = (PartitionRangeReadCommand) readCoordinator.maybeAllowOutOfRangeReads(rangeCommand, replicaPlan.consistencyLevel());
+        // If enabled, request repaired data tracking info from full replicas, but
+        // only if there are multiple full replicas to compare results from.
+        boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled()
+                                      && replicaPlan.contacts().filter(Replica::isFull).size() > 1;
+
+        ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
+        ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair =
+        ReadRepair.create(readCoordinator, command, sharedReplicaPlan, requestTime);
+        DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver =
+        new DataResolver<>(readCoordinator, rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus);
+        ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
+        new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
+
+        if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf() && readCoordinator.localReadSupported())
+        {
+            Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime, trackRepairedStatus));
+        }
+        else
+        {
+            for (Replica replica : replicaPlan.contacts())
+            {
+                Tracing.trace("Enqueuing request to {}", replica);
+                ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
+                Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
+                readCoordinator.sendReadCommand(message, replica.endpoint(), handler);
+            }
+        }
+        return new SingleRangeResponse(resolver, handler, readRepair);
+    }
+
+
     /**
      * Queries the provided sub-range.
      *
@@ -196,56 +243,88 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
      * {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in
      * that it's the query that "continues" whatever we're previously queried).
      */
-    private IRangeResponse query(ReplicaPlan.ForRangeRead replicaPlan, ReadCoordinator readCoordinator, boolean isFirst)
+    private PartitionIterator query(ClusterMetadata cm, ReplicaPlan.ForRangeRead replicaPlan, ReadCoordinator readCoordinator, List<ReadRepair<?, ?>> readRepairs, boolean isFirst)
     {
         PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), isFirst);
-        
-        // If enabled, request repaired data tracking info from full replicas, but
-        // only if there are multiple full replicas to compare results from.
-        boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled()
-                                      && replicaPlan.contacts().filter(Replica::isFull).size() > 1;
 
-        ClusterMetadata cm = ClusterMetadata.current();
-        TableMetadata metadata = command.metadata();
-        TableParams tableParams = metadata.params;
-        TransactionalMode transactionalMode = tableParams.transactionalMode;
-        TransactionalMigrationFromMode transactionalMigrationFromMode = tableParams.transactionalMigrationFrom;
-        if (transactionalMigrationFromMode != TransactionalMigrationFromMode.none && transactionalMode.nonSerialReadsThroughAccord && transactionalMigrationFromMode.nonSerialWritesThroughAccord() && transactionalMigrationFromMode.nonSerialReadsThroughAccord())
-            throw new UnsupportedOperationException("Live migration is not supported, can't safely read when migrating from " + transactionalMigrationFromMode + " to " + transactionalMode);
-        if (transactionalMode.nonSerialReadsThroughAccord && readCoordinator.isEventuallyConsistent())
+        // Accord interop execution should always be coordinated through the C* plumbing
+        if (!readCoordinator.isEventuallyConsistent())
         {
-            //TODO (nicetohave): This is very inefficient because it will not map the the command store owned ranges
-            // so every command store will return results and most will be discarded due to the limit
-            // Really we want to split the ranges by command stores owned ranges and then query one at a time
-            AsyncTxnResult result = StorageProxy.readWithAccord(cm, rangeCommand, ImmutableList.of(rangeCommand.dataRange().keyRange()), replicaPlan.consistencyLevel(), requestTime);
-            return new AccordRangeResponse(result, rangeCommand.isReversed(), replicaPlan.consistencyLevel(), requestTime);
+            SingleRangeResponse response = executeNormal(replicaPlan, rangeCommand, readCoordinator);
+            readRepairs.add(response.getReadRepair());
+            return response;
         }
-        else
-        {
-            ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan);
-            ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair =
-            ReadRepair.create(ReadCoordinator.DEFAULT, command, sharedReplicaPlan, requestTime);
-            DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver =
-            new DataResolver<>(ReadCoordinator.DEFAULT, rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus);
-            ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
-            new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
 
-            if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf())
+        List<RangeReadWithTarget> reads = ConsensusRequestRouter.splitReadIntoAccordAndNormal(cm, rangeCommand, readCoordinator, requestTime);
+        // Special case returning directly to avoid wrapping the iterator and applying the limits an extra time
+        if (reads.size() == 1)
+        {
+            RangeReadWithTarget rangeReadWithTarget = reads.get(0);
+            checkState(rangeReadWithTarget.read.dataRange().keyRange().equals(rangeCommand.dataRange().keyRange()));
+            if (rangeReadWithTarget.target == RangeReadTarget.accord && readCoordinator.isEventuallyConsistent())
             {
-                Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime, trackRepairedStatus));
+                return executeAccord(cm,
+                                     rangeReadWithTarget.read,
+                                     replicaPlan.consistencyLevel());
             }
             else
             {
-                for (Replica replica : replicaPlan.contacts())
-                {
-                    Tracing.trace("Enqueuing request to {}", replica);
-                    ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
-                    Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
-                    MessagingService.instance().sendWithCallback(message, replica.endpoint(), handler);
-                }
+                SingleRangeResponse response = executeNormal(replicaPlan, rangeReadWithTarget.read, readCoordinator);
+                readRepairs.add(response.getReadRepair());
+                return response;
             }
-            return new CassandraRangeResponse(resolver, handler, readRepair);
         }
+
+        // TODO (review): Should this be reworked to execute the queries serially from the iterator? It would respect
+        // any provided limits better but the number of queries created will generally be low (2-3)
+        List<PartitionIterator> responses = new ArrayList<>(reads.size() + 1);
+        // Dummy iterator that checks all the responses for retry on different system hasNext so we don't read
+        // from the first iterator when the second needs to be retried because the split was wrong
+        responses.add(new PartitionIterator()
+        {
+            @Override
+            public void close()
+            {
+
+            }
+
+            @Override
+            public boolean hasNext()
+            {
+                for (int i = 1; i < responses.size(); i++)
+                    responses.get(i).hasNext();
+                return false;
+            }
+
+            @Override
+            public RowIterator next()
+            {
+                throw new NoSuchElementException();
+            }
+        });
+
+        for (RangeReadWithTarget rangeReadWithTarget : reads)
+        {
+            if (rangeReadWithTarget.target == RangeReadTarget.accord && readCoordinator.isEventuallyConsistent())
+                responses.add(executeAccord(cm, rangeReadWithTarget.read, replicaPlan.consistencyLevel()));
+            else
+            {
+                SingleRangeResponse response = executeNormal(replicaPlan, rangeReadWithTarget.read, readCoordinator);
+                responses.add(response);
+                readRepairs.add(response.getReadRepair());
+            }
+        }
+
+        /*
+         * We have to apply limits here if the query spans different systems because each subquery we created
+         * could have gaps in the results since the limit is pushed down independently to each subquery.
+         * So if we don't meet the limit in the first subquery, it's not safe to go to the next one unless
+         * we fully exhausted the data the first subquery might have reached
+         */
+        return command.limits().filter(PartitionIterators.concat(responses),
+                                       0,
+                                       command.selectsFullPartition(),
+                                       command.metadata().enforceStrictLiveness());
     }
 
     PartitionIterator sendNextRequests()
@@ -253,15 +332,26 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
         List<PartitionIterator> concurrentQueries = new ArrayList<>(concurrencyFactor);
         List<ReadRepair<?, ?>> readRepairs = new ArrayList<>(concurrencyFactor);
 
+        ClusterMetadata cm = ClusterMetadata.current();
         try
         {
             for (int i = 0; i < concurrencyFactor && replicaPlans.hasNext(); )
             {
                 ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
-
-                IRangeResponse response = query(replicaPlan, readCoordinator, i == 0);
+                boolean isFirst = i == 0;
+                PartitionIterator response;
+                // Only add the retry wrapper to reroute for the top level coordinator execution
+                // not Accord's interop execution
+                if (readCoordinator.isEventuallyConsistent())
+                {
+                    Function<ClusterMetadata, PartitionIterator> querySupplier = clusterMetadata -> query(clusterMetadata, replicaPlan, readCoordinator, readRepairs, isFirst);
+                    response = retryingPartitionIterator(querySupplier, replicaPlan.consistencyLevel());
+                }
+                else
+                {
+                    response = query(cm, replicaPlan, readCoordinator, readRepairs, isFirst);
+                }
                 concurrentQueries.add(response);
-                readRepairs.add(response.getReadRepair());
                 // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor.
                 rangesQueried += replicaPlan.vnodeCount();
                 i += replicaPlan.vnodeCount();
@@ -283,6 +373,57 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
         return counter.applyTo(StorageProxy.concatAndBlockOnRepair(concurrentQueries, readRepairs));
     }
 
+    // Wrap the iterator to retry if request routing is incorrect
+    private PartitionIterator retryingPartitionIterator(Function<ClusterMetadata, PartitionIterator> attempt, ConsistencyLevel cl)
+    {
+        return new PartitionIterator()
+        {
+            private ClusterMetadata lastClusterMetadata = ClusterMetadata.current();
+            private PartitionIterator delegate = attempt.apply(lastClusterMetadata);
+
+            @Override
+            public void close()
+            {
+                delegate.close();
+            }
+
+            @Override
+            public boolean hasNext()
+            {
+                while (true)
+                {
+                    try
+                    {
+                        return delegate.hasNext();
+                    }
+                    catch (RetryOnDifferentSystemException e)
+                    {
+                        readMetrics.retryDifferentSystem.mark();
+                        readMetricsForLevel(cl).retryDifferentSystem.mark();
+                        logger.debug("Retrying range read on different system because some reads were misrouted according to Accord");
+                        Tracing.trace("Got {} from range reads, will retry", e);
+                    }
+                    catch (CoordinatorBehindException e)
+                    {
+                        readMetrics.retryCoordinatorBehind.mark();
+                        readMetricsForLevel(cl).retryCoordinatorBehind.mark();
+                        logger.debug("Retrying range read now that coordinator has caught up to cluster metadata");
+                        Tracing.trace("Got {} from range reads, will retry", e);
+                    }
+                    // Fetch the next epoch to retry
+                    lastClusterMetadata = ClusterMetadata.current();
+                    delegate = attempt.apply(lastClusterMetadata);
+                }
+            }
+
+            @Override
+            public RowIterator next()
+            {
+                return delegate.next();
+            }
+        };
+    }
+
     @Override
     public void close()
     {

src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java

@@ -112,6 +112,6 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator
         Tracing.trace("Submitted scanning all ranges requests to {} nodes", nodes);
 
         // skip read-repair for top-k query because data mismatch may be caused by top-k algorithm instead of actual inconsistency.
-        return new CassandraRangeResponse(resolver, handler, NoopReadRepair.instance);
+        return new SingleRangeResponse(resolver, handler, NoopReadRepair.instance);
     }
 }

src/java/org/apache/cassandra/service/reads/range/SingleRangeResponse.java

@@ -23,12 +23,13 @@ import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.locator.EndpointsForRange;
 import org.apache.cassandra.locator.ReplicaPlan;
+import org.apache.cassandra.locator.ReplicaPlan.ForRangeRead;
 import org.apache.cassandra.service.reads.DataResolver;
 import org.apache.cassandra.service.reads.ReadCallback;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.utils.AbstractIterator;
 
-class CassandraRangeResponse extends AbstractIterator<RowIterator> implements IRangeResponse
+class SingleRangeResponse extends AbstractIterator<RowIterator> implements PartitionIterator
 {
     private final DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver;
     private final ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler;
@@ -36,17 +37,16 @@ class CassandraRangeResponse extends AbstractIterator<RowIterator> implements IR
 
     private PartitionIterator result;
 
-    CassandraRangeResponse(DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver,
-                           ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler,
-                           ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair)
+    SingleRangeResponse(DataResolver<EndpointsForRange, ReplicaPlan.ForRangeRead> resolver,
+                        ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler,
+                        ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> readRepair)
     {
         this.resolver = resolver;
         this.handler = handler;
         this.readRepair = readRepair;
     }
 
-    @Override
-    public ReadRepair<EndpointsForRange, ReplicaPlan.ForRangeRead> getReadRepair()
+    public ReadRepair<EndpointsForRange, ForRangeRead> getReadRepair()
     {
         return readRepair;
     }

src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java

@@ -94,7 +94,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
 
     void sendReadCommand(Replica to, ReadCallback<E, P> readCallback, boolean speculative, boolean trackRepairedStatus)
     {
-        ReadCommand command = coordinator.maybeAllowOutOfRangeReads(this.command);
+        ReadCommand command = coordinator.maybeAllowOutOfRangeReads(this.command, replicaPlan().consistencyLevel());
 
         if (to.isSelf() && coordinator.localReadSupported())
         {

src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java

@@ -53,10 +53,10 @@ import org.apache.cassandra.utils.concurrent.CountDownLatch;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Iterables.all;
 import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
-import static com.google.common.collect.Iterables.all;
 
 public class BlockingPartitionRepair
         extends AsyncFuture<Object> implements RequestCallback<Object>, PendingPartitionRepair
@@ -155,7 +155,7 @@ public class BlockingPartitionRepair
     @VisibleForTesting
     protected void sendRR(Message<Mutation> message, InetAddressAndPort endpoint)
     {
-        checkArgument(message.payload.allowsPotentialTransactionConflicts() == coordinator.allowsPotentialTransactionConflicts(), "Mutation allowing transaction conflicts should match coordinator");
+        checkArgument(message.payload.potentialTxnConflicts() == coordinator.potentialTxnConflicts(), "Mutation allowing transaction conflicts should match coordinator");
         coordinator.sendReadRepairMutation(message, endpoint, this);
     }
 
@@ -241,7 +241,7 @@ public class BlockingPartitionRepair
 
             if (mutation == null)
             {
-                mutation = BlockingReadRepairs.createRepairMutation(update, repairPlan.consistencyLevel(), replica.endpoint(), true, coordinator.allowsPotentialTransactionConflicts());
+                mutation = BlockingReadRepairs.createRepairMutation(update, repairPlan.consistencyLevel(), replica.endpoint(), true, coordinator.potentialTxnConflicts());
                 versionedMutations[versionIdx] = mutation;
             }