Schema based accord fast path configuration

Patch by Blake Eggleston; Reviewed by David Capwell and Alex Petrov for CASSANDRA-19009

conf/cassandra.yaml

@@ -2661,3 +2661,6 @@ storage_compatibility_mode: NONE
 #
 #  # Progress log scheduling delay
 #  progress_log_schedule_delay: 1s
+#
+#  # how quickly the fast path is reconfigured when nodes go up/down
+#  fast_path_update_delay: 5s

modules/accord

@@ -1 +1 @@
-Subproject commit 9f21a24660fe49881e0131813f9eff850e25b3dc
+Subproject commit 5523cfefef163efee53c8cc57595f5b50ea4f363

src/java/org/apache/cassandra/config/AccordSpec.java

@@ -42,4 +42,6 @@ public class AccordSpec
     public DurationSpec.IntMillisecondsBound barrier_max_backoff = new DurationSpec.IntMillisecondsBound("10m");
 
     public DurationSpec.IntMillisecondsBound range_barrier_timeout = new DurationSpec.IntMillisecondsBound("2m");
+
+    public volatile DurationSpec fast_path_update_delay = new DurationSpec.IntSecondsBound(5);
 }

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -5313,6 +5313,16 @@ public class DatabaseDescriptor
         return conf.accord.shard_count.or(DatabaseDescriptor::getAvailableProcessors);
     }
 
+    public static long getAccordFastPathUpdateDelayMillis()
+    {
+        return conf.accord.fast_path_update_delay.to(TimeUnit.MILLISECONDS);
+    }
+
+    public static void setAccordFastPathUpdateDelayMillis(long millis)
+    {
+        conf.accord.fast_path_update_delay = new DurationSpec.IntMillisecondsBound(millis);
+    }
+
     public static boolean getForceNewPreparedStatementBehaviour()
     {
         return conf.force_new_prepared_statement_behaviour;

src/java/org/apache/cassandra/cql3/statements/TransactionStatement.java

@@ -371,7 +371,7 @@ public class TransactionStatement implements CQLStatement.CompositeCQLStatement,
 
             Txn txn = createTxn(state.getClientState(), options);
 
-            AccordService.instance().maybeConvertKeyspacesToAccord(txn);
+            AccordService.instance().maybeConvertTablesToAccord(txn);
 
             TxnResult txnResult = AccordService.instance().coordinate(txn, options.getConsistency(), requestTime);
             if (txnResult.kind() == retry_new_protocol)

src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java

@@ -23,9 +23,11 @@ import com.google.common.collect.ImmutableSet;
 
 import org.apache.cassandra.cql3.statements.PropertyDefinitions;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.KeyspaceParams.Option;
 import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 
 public final class KeyspaceAttributes extends PropertyDefinitions
 {
@@ -48,6 +50,10 @@ public final class KeyspaceAttributes extends PropertyDefinitions
         Map<String, String> replicationOptions = getAllReplicationOptions();
         if (!replicationOptions.isEmpty() && !replicationOptions.containsKey(ReplicationParams.CLASS))
             throw new ConfigurationException("Missing replication strategy class");
+
+        FastPathStrategy strategy = getFastPathStrategy();
+        if (strategy != null && strategy.kind() == FastPathStrategy.Kind.INHERIT_KEYSPACE)
+            throw new ConfigurationException("Cannot use keyspace inheriting fast path strategy with keyspaces");
     }
 
     public String getReplicationStrategyClass()
@@ -63,10 +69,26 @@ public final class KeyspaceAttributes extends PropertyDefinitions
              : replication;
     }
 
+    private FastPathStrategy getFastPathStrategy()
+    {
+        if (!hasOption(Option.FAST_PATH))
+            return null;
+
+        try
+        {
+            return FastPathStrategy.fromMap(getMap(Option.FAST_PATH.toString()));
+        }
+        catch (SyntaxException e)
+        {
+            return FastPathStrategy.keyspaceStrategyFromString(getString(Option.FAST_PATH.toString()));
+        }
+    }
+
     KeyspaceParams asNewKeyspaceParams()
     {
         boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), KeyspaceParams.DEFAULT_DURABLE_WRITES);
-        return KeyspaceParams.create(durableWrites, getAllReplicationOptions());
+        FastPathStrategy fastPath = getFastPathStrategy();
+        return KeyspaceParams.create(durableWrites, getAllReplicationOptions(), fastPath != null ? fastPath : FastPathStrategy.simple());
     }
 
     KeyspaceParams asAlteredKeyspaceParams(KeyspaceParams previous)
@@ -76,7 +98,8 @@ public final class KeyspaceAttributes extends PropertyDefinitions
         ReplicationParams replication = getReplicationStrategyClass() == null
                                       ? previous.replication
                                       : ReplicationParams.fromMapWithDefaults(getAllReplicationOptions(), previousOptions);
-        return new KeyspaceParams(durableWrites, replication);
+        FastPathStrategy fastPath = getFastPathStrategy();
+        return new KeyspaceParams(durableWrites, replication, fastPath != null ? fastPath : previous.fastPath);
     }
 
     public boolean hasOption(Option option)

src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java

@@ -25,6 +25,7 @@ import com.google.common.collect.Sets;
 
 import org.apache.cassandra.cql3.statements.PropertyDefinitions;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.SyntaxException;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.schema.CompressionParams;
@@ -32,6 +33,7 @@ import org.apache.cassandra.schema.MemtableParams;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableParams;
 import org.apache.cassandra.schema.TableParams.Option;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 
@@ -151,6 +153,18 @@ public final class TableAttributes extends PropertyDefinitions
         if (hasOption(READ_REPAIR))
             builder.readRepair(ReadRepairStrategy.fromString(getString(READ_REPAIR)));
 
+        if (hasOption(Option.FAST_PATH))
+        {
+            try
+            {
+                builder.fastPath(FastPathStrategy.fromMap(getMap(Option.FAST_PATH)));
+            }
+            catch (SyntaxException e)
+            {
+                builder.fastPath(FastPathStrategy.tableStrategyFromString(getString(Option.FAST_PATH)));
+            }
+        }
+
         return builder.build();
     }
 

src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java

@@ -246,7 +246,7 @@ public class CassandraStreamReceiver implements StreamReceiver
         checkNotNull(minVersion, "Unable to determine minimum cluster version");
         IAccordService accordService = AccordService.instance();
         if (session.streamOperation().requiresBarrierTransaction()
-            && accordService.isAccordManagedKeyspace(cfs.keyspace.getName())
+            && accordService.isAccordManagedTable(cfs.getTableId())
             && CassandraVersion.CASSANDRA_5_0.compareTo(minVersion) >= 0)
             accordService.postStreamReceivingBarrier(cfs, ranges);
 

src/java/org/apache/cassandra/dht/AccordSplitter.java

@@ -21,6 +21,7 @@ package org.apache.cassandra.dht;
 import java.math.BigInteger;
 
 import accord.local.ShardDistributor;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
@@ -54,9 +55,9 @@ public abstract class AccordSplitter implements ShardDistributor.EvenSplit.Split
         BigInteger end = endBound instanceof SentinelKey ? maximumValue() : valueForToken(endBound.token());
         BigInteger sizeOfRange = end.subtract(start);
 
-        String keyspace = startBound.keyspace();
-        return new TokenRange(startOffset.equals(ZERO) ? startBound : new TokenKey(keyspace, tokenForValue(start.add(startOffset))),
-                              endOffset.compareTo(sizeOfRange) >= 0 ? endBound : new TokenKey(keyspace, tokenForValue(start.add(endOffset))));
+        TableId tableId = startBound.table();
+        return new TokenRange(startOffset.equals(ZERO) ? startBound : new TokenKey(tableId, tokenForValue(start.add(startOffset))),
+                              endOffset.compareTo(sizeOfRange) >= 0 ? endBound : new TokenKey(tableId, tokenForValue(start.add(endOffset))));
     }
 
     @Override

src/java/org/apache/cassandra/dht/BootStrapper.java

@@ -40,7 +40,6 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.streaming.StreamEvent;
 import org.apache.cassandra.streaming.StreamEventHandler;
 import org.apache.cassandra.streaming.StreamOperation;
@@ -127,7 +126,8 @@ public class BootStrapper extends ProgressEventNotifierSupport
                                                    true,
                                                    DatabaseDescriptor.getStreamingConnectionsPerHost(),
                                                    movements,
-                                                   strictMovements);
+                                                   strictMovements,
+                                                   true);
 
         if (beingReplaced != null)
             streamer.addSourceFilter(new RangeStreamer.ExcludedSourcesFilter(Collections.singleton(beingReplaced)));
@@ -137,8 +137,6 @@ public class BootStrapper extends ProgressEventNotifierSupport
             logger.debug("Schema does not contain any non-local keyspaces to stream on bootstrap");
         for (String keyspaceName : nonLocalStrategyKeyspaces)
         {
-            if (AccordService.instance().isAccordManagedKeyspace(keyspaceName))
-                continue;
             KeyspaceMetadata ksm = metadata.schema.getKeyspaces().get(keyspaceName).get();
             if (ksm.params.replication.isMeta())
                 continue;

src/java/org/apache/cassandra/dht/RangeStreamer.java

@@ -59,7 +59,9 @@ import org.apache.cassandra.locator.ReplicaCollection;
 import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
 import org.apache.cassandra.locator.Replicas;
 import org.apache.cassandra.locator.NodeProximity;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.streaming.PreviewKind;
 import org.apache.cassandra.streaming.StreamOperation;
 import org.apache.cassandra.streaming.StreamPlan;
@@ -99,6 +101,7 @@ public class RangeStreamer
     private final StreamStateStore stateStore;
     private final MovementMap movements;
     private final MovementMap strictMovements;
+    private final boolean excludeAccordTables;
 
     public static class FetchReplica
     {
@@ -299,10 +302,11 @@ public class RangeStreamer
                          boolean connectSequentially,
                          int connectionsPerHost,
                          MovementMap movements,
-                         MovementMap strictMovements)
+                         MovementMap strictMovements,
+                         boolean excludeAccordTables)
     {
         this(metadata, streamOperation, useStrictConsistency, proximity, stateStore,
-             FailureDetector.instance, connectSequentially, connectionsPerHost, movements, strictMovements);
+             FailureDetector.instance, connectSequentially, connectionsPerHost, movements, strictMovements, excludeAccordTables);
     }
 
     RangeStreamer(ClusterMetadata metadata,
@@ -314,8 +318,10 @@ public class RangeStreamer
                   boolean connectSequentially,
                   int connectionsPerHost,
                   MovementMap movements,
-                  MovementMap strictMovements)
+                  MovementMap strictMovements,
+                  boolean excludeAccordTables)
     {
+        this.excludeAccordTables = excludeAccordTables;
         Preconditions.checkArgument(streamOperation == StreamOperation.BOOTSTRAP || streamOperation == StreamOperation.REBUILD, streamOperation);
         this.metadata = metadata;
         this.description = streamOperation.getDescription();
@@ -760,8 +766,17 @@ public class RangeStreamer
                 logger.debug("Source and our replicas {}", fetchReplicas);
                 logger.debug("Source {} Keyspace {}  streaming full {} transient {}", source, keyspace, full, transientReplicas);
 
-                /* Send messages to respective folks to stream data over to me */
-                streamPlan.requestRanges(source, keyspace, full, transientReplicas);
+                KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspace);
+                if (excludeAccordTables && StreamPlan.hasAccordTables(ksm))
+                {
+                    String[] cfNames = StreamPlan.nonAccordTablesForKeyspace(ksm);
+                    if (cfNames != null)
+                        streamPlan.requestRanges(source, keyspace, full, transientReplicas, cfNames);
+                }
+                else
+                {
+                    streamPlan.requestRanges(source, keyspace, full, transientReplicas);
+                }
             });
         });
 

src/java/org/apache/cassandra/locator/ReplicaLayout.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.FBUtilities;
@@ -357,11 +358,11 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
      * @return the read layout for a token - this includes natural replicas, i.e. those that are not pending.
      * They are reverse sorted by the badness score of the configured snitch
      */
-    static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, Token token, ReadCoordinator coordinator)
+    static ReplicaLayout.ForTokenRead forTokenReadSorted(ClusterMetadata metadata, Keyspace keyspace, AbstractReplicationStrategy replicationStrategy, TableId tableId, Token token, ReadCoordinator coordinator)
     {
         EndpointsForToken replicas = keyspace.getMetadata().params.replication.isLocal()
                                      ? forLocalStrategyToken(metadata, replicationStrategy, token)
-                                     : coordinator.forNonLocalStrategyTokenRead(metadata, keyspace.getMetadata(), token);
+                                     : coordinator.forNonLocalStrategyTokenRead(metadata, keyspace.getMetadata(), tableId, token);
 
         replicas = DatabaseDescriptor.getNodeProximity().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas);
 

src/java/org/apache/cassandra/locator/ReplicaPlans.java

@@ -62,6 +62,7 @@ import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.tcm.ClusterMetadata;
@@ -534,7 +535,7 @@ public class ReplicaPlans
     }
 
 
-    public static ReplicaPlan.ForWrite forReadRepair(ReplicaPlan<?, ?> forRead, ClusterMetadata metadata, Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Predicate<Replica> isAlive, ReadCoordinator coordinator) throws UnavailableException
+    public static ReplicaPlan.ForWrite forReadRepair(ReplicaPlan<?, ?> forRead, ClusterMetadata metadata, Keyspace keyspace, TableId tableId, ConsistencyLevel consistencyLevel, Token token, Predicate<Replica> isAlive, ReadCoordinator coordinator) throws UnavailableException
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
         Selector selector = writeReadRepair(forRead);
@@ -551,7 +552,7 @@ public class ReplicaPlans
                                         liveAndDown.all(),
                                         live.all(),
                                         contacts,
-                                        (newClusterMetadata) -> forReadRepair(forRead, newClusterMetadata, keyspace, consistencyLevel, token, isAlive, coordinator),
+                                        (newClusterMetadata) -> forReadRepair(forRead, newClusterMetadata, keyspace, tableId, consistencyLevel, token, isAlive, coordinator),
                                         metadata.epoch);
     }
 
@@ -880,28 +881,31 @@ public class ReplicaPlans
      * it would break EACH_QUORUM to do so without further filtering
      */
     public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace,
+                                                   TableId tableId,
                                                    Token token,
                                                    @Nullable Index.QueryPlan indexQueryPlan,
                                                    ConsistencyLevel consistencyLevel,
                                                    SpeculativeRetryPolicy retry,
                                                    ReadCoordinator coordinator)
     {
-        return forRead(ClusterMetadata.current(), keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, false);
+        return forRead(ClusterMetadata.current(), keyspace, tableId, token, indexQueryPlan, consistencyLevel, retry, coordinator, false);
     }
 
     public static ReplicaPlan.ForTokenRead forRead(ClusterMetadata metadata,
                                                    Keyspace keyspace,
+                                                   TableId tableId,
                                                    Token token,
                                                    @Nullable Index.QueryPlan indexQueryPlan,
                                                    ConsistencyLevel consistencyLevel,
                                                    SpeculativeRetryPolicy retry,
                                                    ReadCoordinator coordinator)
     {
-        return forRead(metadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, true);
+        return forRead(metadata, keyspace, tableId, token, indexQueryPlan, consistencyLevel, retry, coordinator, true);
     }
 
     private static ReplicaPlan.ForTokenRead forRead(ClusterMetadata metadata,
                                                     Keyspace keyspace,
+                                                    TableId tableId,
                                                     Token token,
                                                     @Nullable Index.QueryPlan indexQueryPlan,
                                                     ConsistencyLevel consistencyLevel,
@@ -910,7 +914,7 @@ public class ReplicaPlans
                                                     boolean throwOnInsufficientLiveReplicas)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        ReplicaLayout.ForTokenRead forTokenReadLiveAndDown = ReplicaLayout.forTokenReadSorted(metadata, keyspace, replicationStrategy, token, coordinator);
+        ReplicaLayout.ForTokenRead forTokenReadLiveAndDown = ReplicaLayout.forTokenReadSorted(metadata, keyspace, replicationStrategy, tableId, token, coordinator);
         ReplicaLayout.ForTokenRead forTokenReadLive = forTokenReadLiveAndDown.filter(FailureDetector.isReplicaAlive);
         EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, forTokenReadLive.all());
         EndpointsForToken contacts = contactForRead(metadata.locator, replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates);
@@ -919,8 +923,8 @@ public class ReplicaPlans
             assureSufficientLiveReplicasForRead(metadata.locator, replicationStrategy, consistencyLevel, contacts);
 
         return new ReplicaPlan.ForTokenRead(keyspace, replicationStrategy, consistencyLevel, candidates, contacts, forTokenReadLiveAndDown.all(),
-                                            (newClusterMetadata) -> forRead(newClusterMetadata, keyspace, token, indexQueryPlan, consistencyLevel, retry, coordinator, false),
-                                            (self) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, coordinator),
+                                            (newClusterMetadata) -> forRead(newClusterMetadata, keyspace, tableId, token, indexQueryPlan, consistencyLevel, retry, coordinator, false),
+                                            (self) -> forReadRepair(self, metadata, keyspace, tableId, consistencyLevel, token, FailureDetector.isReplicaAlive, coordinator),
                                             metadata.epoch);
     }
 
@@ -932,16 +936,18 @@ public class ReplicaPlans
      * There is no speculation for range read queries at present, so we never 'always speculate' here, and a failed response fails the query.
      */
     public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace,
+                                                        TableId tableId,
                                                         @Nullable Index.QueryPlan indexQueryPlan,
                                                         ConsistencyLevel consistencyLevel,
                                                         AbstractBounds<PartitionPosition> range,
                                                         int vnodeCount)
     {
-        return forRangeRead(ClusterMetadata.current(), keyspace, indexQueryPlan, consistencyLevel, range, vnodeCount, true);
+        return forRangeRead(ClusterMetadata.current(), keyspace, tableId, indexQueryPlan, consistencyLevel, range, vnodeCount, true);
     }
 
     public static ReplicaPlan.ForRangeRead forRangeRead(ClusterMetadata metadata,
                                                         Keyspace keyspace,
+                                                        TableId tableId,
                                                         @Nullable Index.QueryPlan indexQueryPlan,
                                                         ConsistencyLevel consistencyLevel,
                                                         AbstractBounds<PartitionPosition> range,
@@ -965,8 +971,8 @@ public class ReplicaPlans
                                             contacts,
                                             forRangeReadLiveAndDown.all(),
                                             vnodeCount,
-                                            (newClusterMetadata) -> forRangeRead(newClusterMetadata, keyspace, indexQueryPlan, consistencyLevel, range, vnodeCount, false),
-                                            (self, token) -> forReadRepair(self, metadata, keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT),
+                                            (newClusterMetadata) -> forRangeRead(newClusterMetadata, keyspace, tableId, indexQueryPlan, consistencyLevel, range, vnodeCount, false),
+                                            (self, token) -> forReadRepair(self, metadata, keyspace, tableId, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT),
                                             metadata.epoch);
     }
 
@@ -1000,6 +1006,7 @@ public class ReplicaPlans
      */
     public static ReplicaPlan.ForRangeRead maybeMerge(ClusterMetadata metadata,
                                                       Keyspace keyspace,
+                                                      TableId tableId,
                                                       ConsistencyLevel consistencyLevel,
                                                       ReplicaPlan.ForRangeRead left,
                                                       ReplicaPlan.ForRangeRead right)
@@ -1037,6 +1044,7 @@ public class ReplicaPlans
                                             newVnodeCount,
                                             (newClusterMetadata) -> forRangeRead(newClusterMetadata,
                                                                                  keyspace,
+                                                                                 tableId,
                                                                                  null, // TODO (TCM) - we only use the recomputed ForRangeRead to check stillAppliesTo - make sure passing null here is ok
                                                                                  consistencyLevel,
                                                                                  newRange,
@@ -1045,7 +1053,7 @@ public class ReplicaPlans
                                             (self, token) -> {
                                                 // It might happen that the ring has moved forward since the operation has started, but because we'll be recomputing a quorum
                                                 // after the operation is complete, we will catch inconsistencies either way.
-                                                return forReadRepair(self, ClusterMetadata.current(), keyspace, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT);
+                                                return forReadRepair(self, ClusterMetadata.current(), keyspace, tableId, consistencyLevel, token, FailureDetector.isReplicaAlive, ReadCoordinator.DEFAULT);
                                             },
                                             left.epoch);
     }

src/java/org/apache/cassandra/repair/AccordRepairJob.java

@@ -21,6 +21,7 @@ package org.apache.cassandra.repair;
 import java.math.BigInteger;
 import javax.annotation.Nullable;
 
+import org.apache.cassandra.service.accord.AccordTopology;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,7 +32,6 @@ import accord.primitives.Seekables;
 import org.apache.cassandra.dht.AccordSplitter;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.service.accord.AccordService;
-import org.apache.cassandra.service.accord.AccordTopologyUtils;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.ConsensusMigrationRepairResult;
 import org.apache.cassandra.tcm.ClusterMetadata;
@@ -65,7 +65,7 @@ public class AccordRepairJob extends AbstractRepairJob
     {
         super(repairSession, cfname);
         IPartitioner partitioner = desc.ranges.iterator().next().left.getPartitioner();
-        this.ranges = AccordTopologyUtils.toAccordRanges(desc.keyspace, desc.ranges);
+        this.ranges = AccordTopology.toAccordRanges(desc.keyspace, desc.ranges);
         this.splitter = partitioner.accordSplitter().apply(ranges);
     }
 

src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java

@@ -26,8 +26,6 @@ import java.util.function.Supplier;
 
 import com.google.common.collect.ImmutableMap;
 
-import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,6 +34,8 @@ import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.exceptions.CasWriteTimeoutException;
+import org.apache.cassandra.locator.MetaStrategy;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MetadataSnapshots;
@@ -44,6 +44,7 @@ import org.apache.cassandra.tcm.log.Entry;
 import org.apache.cassandra.tcm.log.LogReader;
 import org.apache.cassandra.tcm.log.LogState;
 import org.apache.cassandra.tcm.transformations.cms.PreInitialize;
+import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static org.apache.cassandra.tcm.Epoch.FIRST;
 
@@ -223,7 +224,7 @@ public final class DistributedMetadataLogKeyspace
 
     public static KeyspaceMetadata initialMetadata(Set<String> knownDatacenters)
     {
-        return KeyspaceMetadata.create(SchemaConstants.METADATA_KEYSPACE_NAME, new KeyspaceParams(true, ReplicationParams.simpleMeta(1, knownDatacenters)), Tables.of(Log));
+        return KeyspaceMetadata.create(SchemaConstants.METADATA_KEYSPACE_NAME, new KeyspaceParams(true, ReplicationParams.simpleMeta(1, knownDatacenters), FastPathStrategy.simple()), Tables.of(Log));
     }
 
     public static KeyspaceMetadata initialMetadata(String datacenter)

src/java/org/apache/cassandra/schema/KeyspaceMetadata.java

@@ -49,6 +49,7 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.schema.Tables.TablesDiff;
 import org.apache.cassandra.schema.Types.TypesDiff;
 import org.apache.cassandra.schema.Views.ViewsDiff;
+import org.apache.cassandra.utils.LocalizeString;
 
 import static com.google.common.collect.Iterables.any;
 import static java.lang.String.format;
@@ -364,9 +365,16 @@ public final class KeyspaceMetadata implements SchemaElement
             params.replication.appendCqlTo(builder);
 
             builder.append("  AND durable_writes = ")
-                   .append(params.durableWrites)
-                   .append(';')
-                   .toString();
+                   .append(params.durableWrites);
+
+            if (params.fastPath != null)
+            {
+                builder.append("  AND fast_path = '")
+                       .append(LocalizeString.toLowerCaseLocalized(params.fastPath.toString()))
+                       .append("'");
+            }
+
+            builder.append(';');
         }
         return builder.toString();
     }

src/java/org/apache/cassandra/schema/KeyspaceParams.java

@@ -28,10 +28,12 @@ import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
+import static org.apache.cassandra.tcm.serialization.Version.V2;
 import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized;
 
 /**
@@ -54,7 +56,8 @@ public final class KeyspaceParams
     public enum Option
     {
         DURABLE_WRITES,
-        REPLICATION;
+        REPLICATION,
+        FAST_PATH;
 
         @Override
         public String toString()
@@ -65,41 +68,53 @@ public final class KeyspaceParams
 
     public final boolean durableWrites;
     public final ReplicationParams replication;
+    public final FastPathStrategy fastPath;
 
-    public KeyspaceParams(boolean durableWrites, ReplicationParams replication)
+    public KeyspaceParams(boolean durableWrites, ReplicationParams replication, FastPathStrategy fastPath)
     {
         this.durableWrites = durableWrites;
         this.replication = replication;
+        this.fastPath = fastPath;
+    }
+
+    public static KeyspaceParams create(boolean durableWrites, Map<String, String> replication, FastPathStrategy fastPath)
+    {
+        return new KeyspaceParams(durableWrites, ReplicationParams.fromMap(replication), fastPath);
+    }
+
+    public static KeyspaceParams create(boolean durableWrites, Map<String, String> replication, Map<String, String> fastPath)
+    {
+        return create(durableWrites, replication, FastPathStrategy.fromMap(fastPath));
     }
 
     public static KeyspaceParams create(boolean durableWrites, Map<String, String> replication)
     {
-        return new KeyspaceParams(durableWrites, ReplicationParams.fromMap(replication));
+        return create(durableWrites, replication, FastPathStrategy.simple());
     }
 
     public static KeyspaceParams local()
     {
-        return new KeyspaceParams(DEFAULT_LOCAL_DURABLE_WRITES, ReplicationParams.local());
+        return new KeyspaceParams(DEFAULT_LOCAL_DURABLE_WRITES, ReplicationParams.local(), FastPathStrategy.simple());
     }
 
     public static KeyspaceParams simple(int replicationFactor)
     {
-        return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor));
+        return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple());
     }
 
     public static KeyspaceParams simple(String replicationFactor)
     {
-        return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor));
+        return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple());
     }
 
     public static KeyspaceParams simpleTransient(int replicationFactor)
     {
-        return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor));
+        return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple());
     }
 
     public static KeyspaceParams nts(Object... args)
     {
-        return new KeyspaceParams(true, ReplicationParams.nts(args));
+        return new KeyspaceParams(true, ReplicationParams.nts(args), FastPathStrategy.simple());
     }
 
     public void validate(String name, ClientState state, ClusterMetadata metadata)
@@ -118,13 +133,13 @@ public final class KeyspaceParams
 
         KeyspaceParams p = (KeyspaceParams) o;
 
-        return durableWrites == p.durableWrites && replication.equals(p.replication);
+        return durableWrites == p.durableWrites && replication.equals(p.replication) && fastPath.equals(p.fastPath);
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hashCode(durableWrites, replication);
+        return Objects.hashCode(durableWrites, replication, fastPath);
     }
 
     @Override
@@ -133,6 +148,7 @@ public final class KeyspaceParams
         return MoreObjects.toStringHelper(this)
                           .add(Option.DURABLE_WRITES.toString(), durableWrites)
                           .add(Option.REPLICATION.toString(), replication)
+                          .add(Option.FAST_PATH.toString(), fastPath.toString())
                           .toString();
     }
 
@@ -142,19 +158,25 @@ public final class KeyspaceParams
         {
             ReplicationParams.serializer.serialize(t.replication, out, version);
             out.writeBoolean(t.durableWrites);
+            if (version.isAtLeast(V2))
+                FastPathStrategy.serializer.serialize(t.fastPath, out, version);
         }
 
         public KeyspaceParams deserialize(DataInputPlus in, Version version) throws IOException
         {
             ReplicationParams params = ReplicationParams.serializer.deserialize(in, version);
             boolean durableWrites = in.readBoolean();
-            return new KeyspaceParams(durableWrites, params);
+            FastPathStrategy fastPath = version.isAtLeast(V2)
+                    ? FastPathStrategy.serializer.deserialize(in, version)
+                    : FastPathStrategy.simple();
+            return new KeyspaceParams(durableWrites, params, fastPath);
         }
 
         public long serializedSize(KeyspaceParams t, Version version)
         {
             return ReplicationParams.serializer.serializedSize(t.replication, version) +
-                   TypeSizes.sizeof(t.durableWrites);
+                   TypeSizes.sizeof(t.durableWrites) +
+                   (version.isAtLeast(V2) ? FastPathStrategy.serializer.serializedSize(t.fastPath, version) : 0);
         }
     }
 }

src/java/org/apache/cassandra/schema/SchemaKeyspace.java

@@ -44,6 +44,7 @@ import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.db.partitions.*;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder;
 import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff;
@@ -97,6 +98,7 @@ public final class SchemaKeyspace
               + "keyspace_name text,"
               + "durable_writes boolean,"
               + "replication frozen<map<text, text>>,"
+              + "fast_path frozen<map<text, text>>,"
               + "PRIMARY KEY ((keyspace_name)))");
 
     private static final TableMetadata Tables =
@@ -128,6 +130,7 @@ public final class SchemaKeyspace
               + "additional_write_policy text,"
               + "cdc boolean,"
               + "read_repair text,"
+              + "fast_path frozen<map<text, text>>,"
               + "PRIMARY KEY ((keyspace_name), table_name))");
 
     private static final TableMetadata Columns =
@@ -491,7 +494,8 @@ public final class SchemaKeyspace
                .row()
                .add(KeyspaceParams.Option.DURABLE_WRITES.toString(), params.durableWrites)
                .add(KeyspaceParams.Option.REPLICATION.toString(),
-                    (params.replication.isMeta() ? params.replication.asNonMeta() : params.replication).asMap());
+                    (params.replication.isMeta() ? params.replication.asNonMeta() : params.replication).asMap())
+               .add(KeyspaceParams.Option.FAST_PATH.toString(), params.fastPath.asMap());
 
         return builder;
     }
@@ -551,7 +555,7 @@ public final class SchemaKeyspace
                                               .add("id", table.id.asUUID())
                                               .add("flags", TableMetadata.Flag.toStringSet(table.flags));
 
-        addTableParamsToRowBuilder(table.params, rowBuilder);
+        addTableParamsToRowBuilder(table.params, rowBuilder, false);
 
         if (withColumnsAndTriggers)
         {
@@ -569,7 +573,7 @@ public final class SchemaKeyspace
         }
     }
 
-    private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder)
+    private static void addTableParamsToRowBuilder(TableParams params, Row.SimpleBuilder builder, boolean forView)
     {
         builder.add("bloom_filter_fp_chance", params.bloomFilterFpChance)
                .add("comment", params.comment)
@@ -608,6 +612,9 @@ public final class SchemaKeyspace
         // incremental_backups is enabled, to avoid RTE in pre-4.2 versioned node during upgrades
         if (!params.incrementalBackups)
             builder.add("incremental_backups", false);
+
+        if (DatabaseDescriptor.getAccordTransactionsEnabled() && !forView)
+            builder.add("fast_path", params.fastPath.asMap());
     }
 
     private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder)
@@ -820,7 +827,7 @@ public final class SchemaKeyspace
                                               .add("where_clause", view.whereClause.toCQLString())
                                               .add("id", table.id.asUUID());
 
-        addTableParamsToRowBuilder(table.params, rowBuilder);
+        addTableParamsToRowBuilder(table.params, rowBuilder, true);
 
         if (includeColumns)
         {
@@ -966,9 +973,11 @@ public final class SchemaKeyspace
         UntypedResultSet.Row row = query(query, keyspaceName).one();
         boolean durableWrites = row.getBoolean(KeyspaceParams.Option.DURABLE_WRITES.toString());
         Map<String, String> replication = row.getFrozenTextMap(KeyspaceParams.Option.REPLICATION.toString());
-        KeyspaceParams params = KeyspaceParams.create(durableWrites, replication);
+        Map<String, String> fastPath = row.getFrozenTextMap(KeyspaceParams.Option.FAST_PATH.toString());
+        KeyspaceParams params = KeyspaceParams.create(durableWrites, replication, fastPath);
+
         if (keyspaceName.equals(SchemaConstants.METADATA_KEYSPACE_NAME))
-            params = new KeyspaceParams(params.durableWrites, params.replication.asMeta());
+            params = new KeyspaceParams(params.durableWrites, params.replication.asMeta(), FastPathStrategy.simple());
 
         return params;
     }
@@ -1070,7 +1079,8 @@ public final class SchemaKeyspace
                                                                         SpeculativeRetryPolicy.fromString(row.getString("additional_write_policy")) :
                                                                         SpeculativeRetryPolicy.fromString("99PERCENTILE"))
                                                  .cdc(row.has("cdc") && row.getBoolean("cdc"))
-                                                 .readRepair(getReadRepairStrategy(row));
+                                                 .readRepair(getReadRepairStrategy(row))
+                                                 .fastPath(getFastPathStrategy(row));
 
         // allow_auto_snapshot column was introduced in 4.2
         if (row.has("allow_auto_snapshot"))
@@ -1448,4 +1458,11 @@ public final class SchemaKeyspace
                ? ReadRepairStrategy.fromString(row.getString("read_repair"))
                : ReadRepairStrategy.BLOCKING;
     }
+
+    private static FastPathStrategy getFastPathStrategy(UntypedResultSet.Row row)
+    {
+        return row.has("fast_path")
+               ? FastPathStrategy.fromMap(row.getFrozenTextMap("fast_path"))
+               : FastPathStrategy.inheritKeyspace();
+    }
 }

src/java/org/apache/cassandra/schema/TableMetadata.java

@@ -70,6 +70,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.serialization.UDTAndFunctionsAwareMetadataSerializer;
@@ -973,6 +974,12 @@ public class TableMetadata implements SchemaElement
             return this;
         }
 
+        public Builder fastPath(FastPathStrategy val)
+        {
+            params.fastPath(val);
+            return this;
+        }
+
         public Builder defaultTimeToLive(int val)
         {
             params.defaultTimeToLive(val);

src/java/org/apache/cassandra/schema/TableParams.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.service.reads.PercentileSpeculativeRetryPolicy;
@@ -69,7 +70,8 @@ public final class TableParams
         ADDITIONAL_WRITE_POLICY,
         CRC_CHECK_CHANCE,
         CDC,
-        READ_REPAIR;
+        READ_REPAIR,
+        FAST_PATH;
 
         @Override
         public String toString()
@@ -97,6 +99,7 @@ public final class TableParams
     public final ImmutableMap<String, ByteBuffer> extensions;
     public final boolean cdc;
     public final ReadRepairStrategy readRepair;
+    public final FastPathStrategy fastPath;
 
     private TableParams(Builder builder)
     {
@@ -121,6 +124,7 @@ public final class TableParams
         extensions = builder.extensions;
         cdc = builder.cdc;
         readRepair = builder.readRepair;
+        fastPath = builder.fastPath;
     }
 
     public static Builder builder()
@@ -148,7 +152,8 @@ public final class TableParams
                             .additionalWritePolicy(params.additionalWritePolicy)
                             .extensions(params.extensions)
                             .cdc(params.cdc)
-                            .readRepair(params.readRepair);
+                            .readRepair(params.readRepair)
+                            .fastPath(params.fastPath);
     }
 
     public Builder unbuild()
@@ -239,7 +244,8 @@ public final class TableParams
             && memtable.equals(p.memtable)
             && extensions.equals(p.extensions)
             && cdc == p.cdc
-            && readRepair == p.readRepair;
+            && readRepair == p.readRepair
+            && fastPath.equals(fastPath);
     }
 
     @Override
@@ -263,7 +269,8 @@ public final class TableParams
                                 memtable,
                                 extensions,
                                 cdc,
-                                readRepair);
+                                readRepair,
+                                fastPath);
     }
 
     @Override
@@ -275,6 +282,7 @@ public final class TableParams
                           .add(ALLOW_AUTO_SNAPSHOT.toString(), allowAutoSnapshot)
                           .add(BLOOM_FILTER_FP_CHANCE.toString(), bloomFilterFpChance)
                           .add(CRC_CHECK_CHANCE.toString(), crcCheckChance)
+                          .add(FAST_PATH.toString(), fastPath)
                           .add(GC_GRACE_SECONDS.toString(), gcGraceSeconds)
                           .add(DEFAULT_TIME_TO_LIVE.toString(), defaultTimeToLive)
                           .add(INCREMENTAL_BACKUPS.toString(), incrementalBackups)
@@ -289,6 +297,7 @@ public final class TableParams
                           .add(EXTENSIONS.toString(), extensions)
                           .add(CDC.toString(), cdc)
                           .add(READ_REPAIR.toString(), readRepair)
+                          .add(Option.FAST_PATH.toString(), fastPath)
                           .toString();
     }
 
@@ -318,8 +327,8 @@ public final class TableParams
 
         if (!isView)
         {
-            builder.append("AND default_time_to_live = ").append(defaultTimeToLive)
-                   .newLine();
+            builder.append("AND fast_path = ").append(fastPath.asCQL()).newLine();
+            builder.append("AND default_time_to_live = ").append(defaultTimeToLive).newLine();
         }
 
         builder.append("AND extensions = ").append(extensions.entrySet()
@@ -364,6 +373,7 @@ public final class TableParams
         private ImmutableMap<String, ByteBuffer> extensions = ImmutableMap.of();
         private boolean cdc;
         private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING;
+        private FastPathStrategy fastPath = FastPathStrategy.inheritKeyspace();
 
         public Builder()
         {
@@ -482,6 +492,12 @@ public final class TableParams
             return this;
         }
 
+        public Builder fastPath(FastPathStrategy val)
+        {
+            fastPath = val;
+            return this;
+        }
+
         public Builder extensions(Map<String, ByteBuffer> val)
         {
             extensions = ImmutableMap.copyOf(val);
@@ -504,7 +520,10 @@ public final class TableParams
             out.writeUTF(t.speculativeRetry.toString());
             out.writeUTF(t.additionalWritePolicy.toString());
             if (version.isAtLeast(Version.V2))
+            {
                 out.writeUTF(t.memtable.configurationKey());
+                FastPathStrategy.serializer.serialize(t.fastPath, out, version);
+            }
             serializeMap(t.caching.asMap(), out);
             serializeMap(t.compaction.asMap(), out);
             serializeMap(t.compression.asMap(), out);
@@ -532,6 +551,7 @@ public final class TableParams
                    .speculativeRetry(SpeculativeRetryPolicy.fromString(in.readUTF()))
                    .additionalWritePolicy(SpeculativeRetryPolicy.fromString(in.readUTF()))
                    .memtable(version.isAtLeast(Version.V2) ? MemtableParams.get(in.readUTF()) : MemtableParams.DEFAULT)
+                   .fastPath(version.isAtLeast(Version.V2) ? FastPathStrategy.serializer.deserialize(in, version) : FastPathStrategy.simple())
                    .caching(CachingParams.fromMap(deserializeMap(in)))
                    .compaction(CompactionParams.fromMap(deserializeMap(in)))
                    .compression(CompressionParams.fromMap(deserializeMap(in)))
@@ -556,6 +576,7 @@ public final class TableParams
                    sizeof(t.speculativeRetry.toString()) +
                    sizeof(t.additionalWritePolicy.toString()) +
                    (version.isAtLeast(Version.V2) ? sizeof(t.memtable.configurationKey()) : 0) +
+                   (version.isAtLeast(Version.V2) ? FastPathStrategy.serializer.serializedSize(t.fastPath, version) : 0) +
                    serializedSizeMap(t.caching.asMap()) +
                    serializedSizeMap(t.compaction.asMap()) +
                    serializedSizeMap(t.compression.asMap()) +

src/java/org/apache/cassandra/service/Rebuild.java

@@ -113,7 +113,8 @@ public class Rebuild
                                                        false,
                                                        DatabaseDescriptor.getStreamingConnectionsPerHost(),
                                                        rebuildMovements,
-                                                       null);
+                                                       null,
+                                                       true);
             if (sourceDc != null)
                 streamer.addSourceFilter(new RangeStreamer.SingleDatacenterFilter(metadata.locator, sourceDc));
 
@@ -123,16 +124,11 @@ public class Rebuild
             if (keyspace == null)
             {
                 for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
-                {
-                    if (AccordService.instance().isAccordManagedKeyspace(keyspaceName))
-                        continue;
                     streamer.addKeyspaceToFetch(keyspaceName);
-                }
             }
             else if (tokens == null)
             {
-                if (!AccordService.instance().isAccordManagedKeyspace(keyspace))
-                    streamer.addKeyspaceToFetch(keyspace);
+                streamer.addKeyspaceToFetch(keyspace);
             }
             else
             {
@@ -159,8 +155,7 @@ public class Rebuild
                     streamer.addSourceFilter(new RangeStreamer.AllowedSourcesFilter(sources));
                 }
 
-                if (!AccordService.instance().isAccordManagedKeyspace(keyspace))
-                    streamer.addKeyspaceToFetch(keyspace);
+                streamer.addKeyspaceToFetch(keyspace);
             }
 
             StreamResultFuture resultFuture = streamer.fetchAsync();

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -378,7 +378,7 @@ public class StorageProxy implements StorageProxyMBean
                                                         clientState,
                                                         nowInSeconds);
                     IAccordService accordService = AccordService.instance();
-                    accordService.maybeConvertKeyspacesToAccord(txn);
+                    accordService.maybeConvertTablesToAccord(txn);
                     TxnResult txnResult = accordService.coordinate(txn,
                                                                    consistencyForPaxos,
                                                                    requestTime);
@@ -1251,7 +1251,7 @@ public class StorageProxy implements StorageProxyMBean
         AccordUpdate update = new TxnUpdate(fragments, TxnCondition.none(), clForCommit);
         Txn.InMemory txn = new Txn.InMemory(Keys.of(partitionKeys), TxnRead.EMPTY, TxnQuery.EMPTY, update);
         IAccordService accordService = AccordService.instance();
-        accordService.maybeConvertKeyspacesToAccord(txn);
+        accordService.maybeConvertTablesToAccord(txn);
         accordService.coordinate(txn, consistencyLevel, requestTime);
     }
 
@@ -2001,7 +2001,7 @@ public class StorageProxy implements StorageProxyMBean
         TxnRead read = TxnRead.createSerialRead(readCommand, consistencyLevel);
         Txn txn = new Txn.InMemory(read.keys(), read, TxnQuery.ALL);
         IAccordService accordService = AccordService.instance();
-        accordService.maybeConvertKeyspacesToAccord(txn);
+        accordService.maybeConvertTablesToAccord(txn);
         TxnResult txnResult = accordService.coordinate(txn, consistencyLevel, requestTime);
         if (txnResult.kind() == retry_new_protocol)
             return RETRY_NEW_PROTOCOL;

src/java/org/apache/cassandra/service/StorageService.java

@@ -4232,11 +4232,24 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     @Override
     public List<String> getAccordManagedKeyspaces()
+    {
+        Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+        return keyspaces.stream().flatMap(ks -> ks.tables.stream())
+                .filter(tbm -> AccordService.instance().isAccordManagedTable(tbm.id))
+                .map(tbm -> tbm.keyspace)
+                .distinct()
+                .sorted()
+                .collect(toList());
+    }
+
+    @Override
+    public List<String> getAccordManagedTables()
     {
         // TODO (review) These are really just the ones Accord is aware of not necessarily managed
-        Set<String> keyspaces = Schema.instance.getNonLocalStrategyKeyspaces().names();
-        return keyspaces.stream()
-                        .filter(AccordService.instance()::isAccordManagedKeyspace)
+        Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
+        return keyspaces.stream().flatMap(ks -> ks.tables.stream())
+                        .filter(tbm -> AccordService.instance().isAccordManagedTable(tbm.id))
+                        .map(tbm -> tbm.keyspace + '.' + tbm.name)
                         .collect(toList());
     }
 

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -1158,6 +1158,7 @@ public interface StorageServiceMBean extends NotificationEmitter
     String listConsensusMigrations(@Nullable Set<String> keyspaceNames, @Nullable Set<String> tableNames, @Nonnull String format);
 
     List<String> getAccordManagedKeyspaces();
+    List<String> getAccordManagedTables();
 
     /** Gets the concurrency settings for processing stages*/
     static class StageConcurrency implements Serializable

src/java/org/apache/cassandra/service/accord/AccordCommandStores.java

@@ -33,6 +33,7 @@ import accord.utils.RandomSource;
 import org.apache.cassandra.cache.CacheSize;
 import org.apache.cassandra.metrics.AccordStateCacheMetrics;
 import org.apache.cassandra.metrics.CacheSizeMetrics;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 
 public class AccordCommandStores extends CommandStores implements CacheSize
@@ -62,15 +63,15 @@ public class AccordCommandStores extends CommandStores implements CacheSize
         if (!super.shouldBootstrap(node, previous, updated, range))
             return false;
         // we see new ranges when a new keyspace is added, so avoid bootstrap in these cases
-        return contains(previous, ((AccordRoutingKey)  range.start()).keyspace());
+        return contains(previous, ((AccordRoutingKey)  range.start()).table());
     }
 
-    private static boolean contains(Topology previous, String searchKeyspace)
+    private static boolean contains(Topology previous, TableId searchTable)
     {
         for (Range range : previous.ranges())
         {
-            String keyspace = ((AccordRoutingKey)  range.start()).keyspace();
-            if (keyspace.equals(searchKeyspace))
+            TableId table = ((AccordRoutingKey)  range.start()).table();
+            if (table.equals(searchTable))
                 return true;
         }
         return false;

src/java/org/apache/cassandra/service/accord/AccordCommandsForKeys.java

@@ -192,9 +192,9 @@ public class AccordCommandsForKeys
         }
 
         // update in memory cfk data with the update results
-        protected void maybeUpdateCommands(CommandsForKeyUpdate update)
+        protected void maybeUpdateCommands(CommandsForKeyUpdate update, AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> cache)
         {
-            CommandsCachingState commands = (CommandsCachingState) commandStore.depsCommandsForKeyCache().getUnsafe(key());
+            CommandsCachingState commands = (CommandsCachingState) cache.getUnsafe(key());
             if (commands == null)
                 return;
 
@@ -214,7 +214,8 @@ public class AccordCommandsForKeys
             Modified<RoutableKey, CommandsForKeyUpdate> modified = (Modified<RoutableKey, CommandsForKeyUpdate>) next;
 
             CommandsForKeyUpdate current = modified.current;
-            maybeUpdateCommands(current);
+            maybeUpdateCommands(current, commandStore.depsCommandsForKeyCache());
+            maybeUpdateCommands(current, commandStore.allCommandsForKeyCache());
 
             // combine in memory updates
             current = CommandsForKeyGroupUpdater.Immutable.merge(modified.original, current, this);

src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java

@@ -153,15 +153,15 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
     }
 
     @Override
-    public Node.Id mappedId(InetAddressAndPort endpoint)
+    public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
     {
-        return Invariants.nonNull(mapping.mappedId(endpoint), "Unable to map address %s to a Node.Id", endpoint);
+        return mapping.mappedIdOrNull(endpoint);
     }
 
     @Override
-    public InetAddressAndPort mappedEndpoint(Node.Id id)
+    public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
     {
-        return Invariants.nonNull(mapping.mappedEndpoint(id), "Unable to map node id %s to a InetAddressAndPort", id);
+        return mapping.mappedEndpointOrNull(id);
     }
 
     @VisibleForTesting
@@ -179,7 +179,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
 
     synchronized void updateMapping(ClusterMetadata metadata)
     {
-        updateMapping(AccordTopologyUtils.directoryToMapping(mapping, metadata.epoch.getEpoch(), metadata.directory));
+        updateMapping(AccordTopology.directoryToMapping(mapping, metadata.epoch.getEpoch(), metadata.directory));
     }
 
     private void reportMetadata(ClusterMetadata metadata)
@@ -188,7 +188,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
             synchronized (AccordConfigurationService.this)
             {
                 updateMapping(metadata);
-                reportTopology(AccordTopologyUtils.createAccordTopology(metadata));
+                reportTopology(AccordTopology.createAccordTopology(metadata));
             }
         });
     }

src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.service.accord;
 
 import accord.local.Node;
+import accord.utils.Invariants;
 import org.apache.cassandra.locator.InetAddressAndPort;
 
 /**
@@ -26,6 +27,16 @@ import org.apache.cassandra.locator.InetAddressAndPort;
  */
 public interface AccordEndpointMapper
 {
-    Node.Id mappedId(InetAddressAndPort endpoint);
-    InetAddressAndPort mappedEndpoint(Node.Id id);
+    Node.Id mappedIdOrNull(InetAddressAndPort endpoint);
+    InetAddressAndPort mappedEndpointOrNull(Node.Id id);
+
+    default Node.Id mappedId(InetAddressAndPort endpoint)
+    {
+        return Invariants.nonNull(mappedIdOrNull(endpoint), "Unable to map address %s to a Node.Id", endpoint);
+    }
+
+    default InetAddressAndPort mappedEndpoint(Node.Id id)
+    {
+        return Invariants.nonNull(mappedEndpointOrNull(id), "Unable to map node id %s to a InetAddressAndPort", id);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordFastPath.java

@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import accord.local.Node;
+import com.google.common.collect.ImmutableMap;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.serializers.TopologySerializers;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.MetadataValue;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+/**
+ * Cluster availability info for services that need a consistent view of availability for a given epoch, such
+ * as accord topology calculation
+ */
+public class AccordFastPath implements MetadataValue<AccordFastPath>
+{
+    public static final AccordFastPath EMPTY = new AccordFastPath(ImmutableMap.of(), Epoch.EMPTY);
+
+    public enum Status
+    {
+        NORMAL, SHUTDOWN, UNAVAILABLE;
+
+        public boolean isUnavailable()
+        {
+            switch (this)
+            {
+                case UNAVAILABLE:
+                case SHUTDOWN:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+
+        public static final MetadataSerializer<Status> serializer = new MetadataSerializer<Status>()
+        {
+            @Override
+            public void serialize(Status status, DataOutputPlus out, Version version) throws IOException
+            {
+                switch (status)
+                {
+                    case NORMAL: out.write(0); break;
+                    case SHUTDOWN: out.write(1); break;
+                    case UNAVAILABLE: out.write(2); break;
+                    default: throw new IllegalStateException("Unhandled status: " + this);
+                }
+            }
+
+            @Override
+            public Status deserialize(DataInputPlus in, Version version) throws IOException
+            {
+                byte b = in.readByte();
+                switch (b)
+                {
+                    case 0: return NORMAL;
+                    case 1: return SHUTDOWN;
+                    case 2: return UNAVAILABLE;
+                    default: throw new IllegalArgumentException("Unhandled status byte: " + b);
+                }
+            }
+
+            @Override
+            public long serializedSize(Status status, Version version)
+            {
+                return TypeSizes.BYTE_SIZE;
+            }
+        };
+    };
+
+    public static class NodeInfo
+    {
+        public final Status status;
+        public final long updated;
+
+        public NodeInfo(Status status, long updated)
+        {
+            this.status = status;
+            this.updated = updated;
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeInfo nodeInfo = (NodeInfo) o;
+            return updated == nodeInfo.updated && status == nodeInfo.status;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(status, updated);
+        }
+
+        private static final MetadataSerializer<NodeInfo> serializer = new MetadataSerializer<NodeInfo>()
+        {
+            @Override
+            public void serialize(NodeInfo info, DataOutputPlus out, Version version) throws IOException
+            {
+                Status.serializer.serialize(info.status, out, version);
+                out.writeUnsignedVInt(info.updated);
+            }
+
+            @Override
+            public NodeInfo deserialize(DataInputPlus in, Version version) throws IOException
+            {
+                return new NodeInfo(Status.serializer.deserialize(in, version), in.readUnsignedVInt());
+            }
+
+            @Override
+            public long serializedSize(NodeInfo info, Version version)
+            {
+                return Status.serializer.serializedSize(info.status, version) + TypeSizes.sizeofUnsignedVInt(info.updated);
+            }
+        };
+    }
+
+    public final ImmutableMap<Node.Id, NodeInfo> info;
+
+    private final Epoch lastModified;
+
+    AccordFastPath(ImmutableMap<Node.Id, NodeInfo> info, Epoch lastModified)
+    {
+        this.info = info;
+        this.lastModified = lastModified;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AccordFastPath that = (AccordFastPath) o;
+        return info.equals(that.info) && lastModified.equals(that.lastModified);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(info, lastModified);
+    }
+
+    public AccordFastPath withoutNode(NodeId tcmId)
+    {
+        Node.Id node = AccordTopology.tcmIdToAccord(tcmId);
+        if (!info.containsKey(node))
+            return this;
+
+        ImmutableMap.Builder<Node.Id, NodeInfo> builder = ImmutableMap.builder();
+        info.forEach((n, info) -> {
+            if (!n.equals(node))
+                builder.put(n, info);
+        });
+        return new AccordFastPath(builder.build(), lastModified);
+    }
+
+    public AccordFastPath withNodeStatusSince(Node.Id node, Status status, long updateTimeMillis, long updateDelayMillis)
+    {
+        NodeInfo current = info.get(node);
+        if (status == Status.SHUTDOWN && current != null)
+        {
+            // nodes report when they're being shutdown and aren't superseded
+            updateTimeMillis = Math.max(updateTimeMillis, current.updated + 1);
+        }
+
+        if (!canUpdateNodeTo(current, status, updateTimeMillis, updateDelayMillis))
+            throw new InvalidRequestException(String.format("cannot transition %s to %s at %s", node, status, updateTimeMillis));
+
+        ImmutableMap.Builder<Node.Id, NodeInfo> builder = ImmutableMap.builder();
+        builder.put(node, new NodeInfo(status, updateTimeMillis));
+        info.forEach((n, info) -> {
+            if (!n.equals(node))
+                builder.put(n, info);
+        });
+        return new AccordFastPath(builder.build(), lastModified);
+    }
+
+    public boolean canUpdateNodeTo(NodeInfo current, Status status, long updateTimeMillis, long updateDelayMillis)
+    {
+        if (current == null)
+            return status != Status.NORMAL;
+
+        if (current.status == status)
+            return false;
+
+        return updateTimeMillis > current.updated + (status == Status.SHUTDOWN ? 0 : updateDelayMillis);
+    }
+
+    public AccordFastPath withLastModified(Epoch epoch)
+    {
+        return new AccordFastPath(info, epoch);
+    }
+
+    public Epoch lastModified()
+    {
+        return lastModified;
+    }
+
+    public ImmutableSet<Node.Id> unavailableIds()
+    {
+        ImmutableSet.Builder<Node.Id> builder = ImmutableSet.builder();
+        info.entrySet().stream()
+                .filter(entry -> entry.getValue().status.isUnavailable())
+                .map(Map.Entry::getKey)
+                .forEach(builder::add);
+        return builder.build();
+    }
+
+    public static final MetadataSerializer<AccordFastPath> serializer = new MetadataSerializer<AccordFastPath>()
+    {
+        private void serializeMap(Map<Node.Id, NodeInfo> map, DataOutputPlus out, Version version) throws IOException
+        {
+            out.writeInt(map.size());
+            for (Map.Entry<Node.Id, NodeInfo> entry : map.entrySet())
+            {
+                TopologySerializers.nodeId.serialize(entry.getKey(), out, version);
+                NodeInfo.serializer.serialize(entry.getValue(), out, version);
+            }
+        }
+
+        public void serialize(AccordFastPath accordFastPath, DataOutputPlus out, Version version) throws IOException
+        {
+            serializeMap(accordFastPath.info, out, version);
+            Epoch.serializer.serialize(accordFastPath.lastModified, out, version);
+        }
+
+        private ImmutableMap<Node.Id, NodeInfo> deserializeMap(DataInputPlus in, Version version) throws IOException
+        {
+            int size = in.readInt();
+            if (size == 0)
+                return ImmutableMap.of();
+
+            ImmutableMap.Builder<Node.Id, NodeInfo> builder = ImmutableMap.builder();
+            for (int i=0; i<size; i++)
+                builder.put(TopologySerializers.nodeId.deserialize(in, version),
+                        NodeInfo.serializer.deserialize(in, version));
+            return builder.build();
+        }
+
+        public AccordFastPath deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            return new AccordFastPath(deserializeMap(in, version),
+                                    Epoch.serializer.deserialize(in, version));
+        }
+
+        private long serializedMapSize(Map<Node.Id, NodeInfo> map, Version version)
+        {
+            long size = TypeSizes.INT_SIZE;
+            for (Map.Entry<Node.Id, NodeInfo> entry : map.entrySet())
+            {
+                size += TopologySerializers.nodeId.serializedSize(entry.getKey(), version);
+                size += NodeInfo.serializer.serializedSize(entry.getValue(), version);
+            }
+            return size;
+        }
+
+        public long serializedSize(AccordFastPath accordFastPath, Version version)
+        {
+            return serializedMapSize(accordFastPath.info, version)
+                 + Epoch.serializer.serializedSize(accordFastPath.lastModified, version);
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java

@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import accord.api.ConfigurationService;
+import accord.local.Node;
+import accord.primitives.Ranges;
+import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.EndpointState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordFastPath.NodeInfo;
+import org.apache.cassandra.service.accord.AccordFastPath.Status;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.listeners.ChangeListener;
+import org.apache.cassandra.tcm.transformations.ReconfigureAccordFastPath;
+import org.apache.cassandra.utils.Clock;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Listens to availability status of peers and updates tcm fast path data accordingly
+ */
+public abstract class AccordFastPathCoordinator implements ChangeListener, ConfigurationService.Listener
+{
+    private static final AsyncResult<Void> SUCCESS = AsyncResults.success(null);
+
+    private static class PeerStatus
+    {
+        final Node.Id peer;
+        final Status status;
+
+        public PeerStatus(Node.Id peer, Status status)
+        {
+            this.peer = peer;
+            this.status = status;
+        }
+
+        boolean shouldUpdateFastPath(AccordFastPath fastPath, long nowMillis, long delayMillis)
+        {
+            NodeInfo info = fastPath.info.get(peer);
+
+            if (info == null)
+                return status != Status.NORMAL;
+
+            if (info.status == status || info.status == Status.SHUTDOWN)
+                return false;
+
+            return nowMillis - info.updated > delayMillis;
+        }
+    }
+
+    private static class Peers
+    {
+        static final Peers EMPTY = new Peers(0, ImmutableSet.of(), Collections.emptyMap());
+        final long epoch;
+        final ImmutableSet<Node.Id> peers;
+        final Map<Node.Id, PeerStatus> statusMap;
+
+        public Peers(long epoch, ImmutableSet<Node.Id> peers, Map<Node.Id, PeerStatus> statusMap)
+        {
+            this.epoch = epoch;
+            this.peers = peers;
+            this.statusMap = statusMap;
+        }
+
+        public boolean contains(Node.Id node)
+        {
+            return peers.contains(node);
+        }
+
+        public static Peers from(Node.Id localId, Topology topology, Peers prev)
+        {
+            Set<Node.Id> peers = new HashSet<>();
+            topology.forEachOn(localId, (shard, index) -> peers.addAll(shard.nodes));
+            peers.remove(localId);
+
+            Map<Node.Id, PeerStatus> statusMap = new HashMap<>();
+            for (Node.Id peer : peers)
+            {
+                PeerStatus status = prev.statusMap.get(peer);
+                if (status != null)
+                    statusMap.put(peer, status);
+            }
+
+            return new Peers(topology.epoch(), ImmutableSet.copyOf(peers), statusMap);
+        }
+
+        public PeerStatus onUpdate(Node.Id node, Status status)
+        {
+            Invariants.checkArgument(contains(node));
+            PeerStatus peerStatus = new PeerStatus(node, status);
+            statusMap.put(node, peerStatus);
+            return peerStatus;
+        }
+
+        public Iterable<PeerStatus> statusIterable()
+        {
+            return statusMap.values();
+        }
+    }
+
+    private boolean receivedShutdownSignal = false;
+    private volatile Epoch startupEpoch = null;
+    private volatile boolean issuedStartupUpdate = false;
+    private boolean hasRegistered = false;
+    private Peers peers = Peers.EMPTY;
+    private final Node.Id localId;
+
+    public AccordFastPathCoordinator(Node.Id localId)
+    {
+        this.localId = localId;
+    }
+
+    private boolean isShutdown(AccordFastPath fastPath)
+    {
+        NodeInfo info = fastPath.info.get(localId);
+        return info != null && info.status == Status.SHUTDOWN;
+    }
+
+    public synchronized void start()
+    {
+        if (hasRegistered)
+            return;
+
+        ClusterMetadata cm = currentMetadata();
+        startupEpoch = cm.epoch;
+        registerAsListener();
+
+        // TODO: start check routine
+
+        hasRegistered = true;
+
+        AccordFastPath fastPath = cm.accordFastPath;
+
+        long updateDelayMillis = getAccordFastPathUpdateDelayMillis();
+        if (isShutdown(fastPath))
+        {
+            updateFastPath(localId, Status.NORMAL, Clock.Global.currentTimeMillis(), updateDelayMillis);
+            issuedStartupUpdate = true;
+        }
+
+        scheduleMaintenanceTask(updateDelayMillis);
+    }
+
+    abstract ClusterMetadata currentMetadata();
+    abstract void registerAsListener();
+    abstract void updateFastPath(Node.Id node, Status status, long updateTimeMillis, long updateDelayMillis);
+    abstract long getAccordFastPathUpdateDelayMillis();
+
+    private static class Impl extends AccordFastPathCoordinator implements IEndpointStateChangeSubscriber
+    {
+        private final AccordConfigurationService configService;
+
+        public Impl(Node.Id localId, AccordConfigurationService configService)
+        {
+            super(localId);
+            this.configService = configService;
+        }
+
+        @Override
+        ClusterMetadata currentMetadata()
+        {
+            return ClusterMetadata.current();
+        }
+
+        @Override
+        void registerAsListener()
+        {
+            Gossiper.instance.register(this);
+            StorageService.instance.addPreShutdownHook(this::onShutdown);
+            configService.registerListener(this);
+        }
+
+        @Override
+        void updateFastPath(Node.Id node, Status status, long updateTimeMillis, long updateDelayMillis)
+        {
+            Stage.MISC.submit(() -> {
+                ClusterMetadataService.instance().commit(new ReconfigureAccordFastPath(node, status, updateTimeMillis, updateDelayMillis),
+                        metadata -> metadata, ((code, message) -> null));
+            });
+        }
+
+        @Override
+        long getAccordFastPathUpdateDelayMillis()
+        {
+            return DatabaseDescriptor.getAccordFastPathUpdateDelayMillis();
+        }
+
+        @Override
+        public void onAlive(InetAddressAndPort endpoint, EndpointState state)
+        {
+            Node.Id node = configService.mappedIdOrNull(endpoint);
+            if (node != null) onAlive(node);
+        }
+
+        @Override
+        public void onDead(InetAddressAndPort endpoint, EndpointState state)
+        {
+            Node.Id node = configService.mappedIdOrNull(endpoint);
+            if (node != null) onDead(node);
+        }
+    }
+
+    public static AccordFastPathCoordinator create(Node.Id localId, AccordConfigurationService configService)
+    {
+        return new Impl(localId, configService);
+    }
+
+    synchronized void maybeUpdateFastPath(Node.Id node, Status status)
+    {
+        long nowMillis = Clock.Global.currentTimeMillis();
+        long delayMillis = getAccordFastPathUpdateDelayMillis();
+
+        // don't schedule updates for nodes we don't share shards with
+        if (!peers.contains(node))
+            return;
+
+        PeerStatus peerStatus = peers.onUpdate(node, status);
+        ClusterMetadata metadata = currentMetadata();
+        if (peerStatus.shouldUpdateFastPath(metadata.accordFastPath, nowMillis, delayMillis))
+            updateFastPath(node, status, nowMillis, delayMillis);
+    }
+
+    private void scheduleMaintenanceTask(long delayMillis)
+    {
+        ScheduledExecutors.scheduledTasks.schedule(this::maintenance, delayMillis, TimeUnit.MILLISECONDS);
+    }
+
+    synchronized void maintenance()
+    {
+        long nowMillis = Clock.Global.currentTimeMillis();
+        long delayMillis = getAccordFastPathUpdateDelayMillis();
+        try
+        {
+            ClusterMetadata metadata = currentMetadata();
+            for (PeerStatus status : peers.statusIterable())
+            {
+                if (status.shouldUpdateFastPath(metadata.accordFastPath, nowMillis, delayMillis))
+                    updateFastPath(status.peer, status.status, nowMillis, delayMillis);
+            }
+        }
+        finally
+        {
+            scheduleMaintenanceTask(delayMillis);
+        }
+    }
+
+    void onAlive(Node.Id node)
+    {
+        maybeUpdateFastPath(node, Status.NORMAL);
+    }
+
+    public void onDead(Node.Id node)
+    {
+        maybeUpdateFastPath(node, Status.UNAVAILABLE);
+    }
+
+    public void onShutdown()
+    {
+        synchronized (this)
+        {
+            receivedShutdownSignal = true;
+        }
+
+        updateFastPath(localId, Status.SHUTDOWN, Clock.Global.currentTimeMillis(), getAccordFastPathUpdateDelayMillis());
+    }
+
+    /**
+     * In case we somehow missed that we've marked ourselves shutdown on startup
+     */
+    @Override
+    public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
+    {
+        if (next.epoch.compareTo(startupEpoch) <= 0)
+            return;
+
+        if (!isShutdown(next.accordFastPath))
+            return;
+
+        synchronized (this)
+        {
+            if (receivedShutdownSignal || issuedStartupUpdate)
+                return;
+            issuedStartupUpdate = true;
+        }
+
+        updateFastPath(localId, Status.NORMAL, Clock.Global.currentTimeMillis(), getAccordFastPathUpdateDelayMillis());
+    }
+
+    synchronized void updatePeers(Topology topology)
+    {
+        if (topology.epoch() <= peers.epoch)
+            return;
+
+        peers = Peers.from(localId, topology, peers);
+    }
+
+    @VisibleForTesting
+    synchronized boolean isPeer(Node.Id node)
+    {
+        return peers.contains(node);
+    }
+
+    @Override
+    public AsyncResult<Void> onTopologyUpdate(Topology topology, boolean startSync)
+    {
+        updatePeers(topology);
+        return SUCCESS;
+    }
+
+    @Override public void onRemoteSyncComplete(Node.Id node, long epoch) {}
+    @Override public void truncateTopologyUntil(long epoch) {}
+    @Override public void onEpochClosed(Ranges ranges, long epoch) {}
+    @Override public void onEpochRedundant(Ranges ranges, long epoch) {}
+}

src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java

@@ -52,7 +52,8 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.RangesAtEndpoint;
-import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.streaming.PreviewKind;
@@ -267,16 +268,16 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
 
                 // TODO (correctness): check epoch
                 // TODO (correctness): handle dropped tables
-                KeyspaceMetadata ksm = ClusterMetadata.current().schema.getKeyspaceMetadata(range.keyspace());
-                Invariants.checkState(ksm != null, "Keyspace %s not found", range.keyspace());
-                Invariants.checkState(ksm.tables.size() > 0, "Keyspace '%s' has no tables", range.keyspace());
+                TableId tableId = range.table();
+                TableMetadata table = ClusterMetadata.current().schema.getKeyspaces().getTableOrViewNullable(tableId);
+                Invariants.checkState(table != null, "Table with id %s not found", tableId);
 
                 // FIXME: may also be relocation
                 StreamPlan plan = new StreamPlan(StreamOperation.BOOTSTRAP, 1, false,
                                                  null, PreviewKind.NONE).flushBeforeTransfer(true);
 
                 RangesAtEndpoint ranges = RangesAtEndpoint.toDummyList(Collections.singleton(range.toKeyspaceRange()));
-                ksm.tables.forEach(table -> plan.transferRanges(to, table.keyspace, ranges, table.name));
+                plan.transferRanges(to, table.keyspace, ranges, table.name);
                 StreamResultFuture future = plan.execute();
                 return AsyncChains.success(StreamData.of(range, future.planId, hasDataToStream(future.getCoordinator(), to)));
             }

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -862,7 +862,7 @@ public class AccordKeyspace
 
     private static ByteBuffer serializeKey(PartitionKey key)
     {
-        return TupleType.pack(ByteBufferAccessor.instance, Arrays.asList(UUIDSerializer.instance.serialize(key.tableId().asUUID()), key.partitionKey().getKey()));
+        return TupleType.pack(ByteBufferAccessor.instance, Arrays.asList(UUIDSerializer.instance.serialize(key.table().asUUID()), key.partitionKey().getKey()));
     }
 
     private static ByteBuffer serializeTimestamp(Timestamp timestamp)
@@ -1303,7 +1303,7 @@ public class AccordKeyspace
         TableMetadata metadata = Schema.instance.getTableMetadata(tableId);
         if (metadata == null)
             throw new IllegalStateException("Table with id " + tableId + " could not be found; was it deleted?");
-        return new PartitionKey(metadata.keyspace, tableId, metadata.partitioner.decorateKey(key));
+        return new PartitionKey(tableId, metadata.partitioner.decorateKey(key));
     }
 
     public static PartitionKey deserializeKey(UntypedResultSet.Row row)
@@ -1358,7 +1358,7 @@ public class AccordKeyspace
         return executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TIMESTAMPS_FOR_KEY),
                                commandStore.id(),
                                serializeToken(key.token()),
-                               key.tableId().asUUID(), key.partitionKey().getKey());
+                               key.table().asUUID(), key.partitionKey().getKey());
     }
 
     public static TimestampsForKey loadTimestampsForKey(AccordCommandStore commandStore, PartitionKey key)

src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord;
 
 import java.nio.ByteBuffer;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.ToLongFunction;
 
 import com.google.common.collect.ImmutableSortedMap;
@@ -58,6 +59,7 @@ import accord.primitives.Txn.Kind;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
 import accord.primitives.Writes;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
@@ -84,7 +86,8 @@ public class AccordObjectSizes
         return ((AccordRoutingKey) key).estimatedSizeOnHeap();
     }
 
-    private static final long EMPTY_RANGE_SIZE = measure(TokenRange.fullRange(""));
+    private static final TableId EMPTY_ID = TableId.fromUUID(new UUID(0, 0));
+    private static final long EMPTY_RANGE_SIZE = measure(TokenRange.fullRange(EMPTY_ID));
     public static long range(Range range)
     {
         return EMPTY_RANGE_SIZE + key(range.start()) + key(range.end());
@@ -271,7 +274,7 @@ public class AccordObjectSizes
 
     private static class CommandEmptySizes
     {
-        private final static TokenKey EMPTY_KEY = new TokenKey("doesnotexist", null);
+        private final static TokenKey EMPTY_KEY = new TokenKey(EMPTY_ID, null);
         private final static TxnId EMPTY_TXNID = new TxnId(42, 42, Kind.Read, Domain.Key, new Node.Id(42));
 
         private static CommonAttributes attrs(boolean hasDeps, boolean hasTxn)

src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java

@@ -68,6 +68,20 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
                '}';
     }
 
+    @Override
+    public boolean hasUpdate()
+    {
+        boolean hasUpdate = AccordSafeState.super.hasUpdate();
+
+        // cfk initialization is legal, but doesn't need to be propagated to the cache (and would
+        // cause an exception to be thrown if it were). Making an exception on the cache side could
+        // throw away applied cfk updates as well, so it's special cased here
+        if (hasUpdate && original == null && current != null && current.commands().isEmpty())
+            return false;
+
+        return hasUpdate;
+    }
+
     @Override
     public AccordCachingState<RoutableKey, CommandsForKey> global()
     {

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -28,6 +28,7 @@ import javax.annotation.Nonnull;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.apache.cassandra.tcm.transformations.AddAccordTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -76,6 +77,7 @@ import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessageDelivery;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
 import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter;
@@ -92,7 +94,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.transformations.AddAccordKeyspace;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.ExecutorUtils;
@@ -122,6 +123,7 @@ public class AccordService implements IAccordService, Shutdownable
     private final Shutdownable nodeShutdown;
     private final AccordMessageSink messageSink;
     private final AccordConfigurationService configService;
+    private final AccordFastPathCoordinator fastPathCoordinator;
     private final AccordScheduler scheduler;
     private final AccordDataStore dataStore;
     private final AccordJournal journal;
@@ -139,7 +141,7 @@ public class AccordService implements IAccordService, Shutdownable
         @Override
         public long barrier(@Nonnull Seekables keysOrRanges, long minEpoch, Dispatcher.RequestTime requestTime, long timeoutNanos, BarrierType barrierType, boolean isForWrite)
         {
-            throw new UnsupportedOperationException("No accord barriers should be executed when accord_transactions_enabled = false in cassandra.yaml");
+            throw new UnsupportedOperationException("No accord barriers should be executed when accord.enabled = false in cassandra.yaml");
         }
 
         @Override
@@ -195,7 +197,7 @@ public class AccordService implements IAccordService, Shutdownable
         public void receive(Message<List<AccordSyncPropagator.Notification>> message) {}
 
         @Override
-        public boolean isAccordManagedKeyspace(String keyspace)
+        public boolean isAccordManagedTable(TableId keyspace)
         {
             return false;
         }
@@ -207,7 +209,7 @@ public class AccordService implements IAccordService, Shutdownable
         }
 
         @Override
-        public void ensureKeyspaceIsAccordManaged(String keyspace) {}
+        public void ensureTableIsAccordManaged(TableId tableId) {}
     };
 
     private static volatile IAccordService instance = null;
@@ -236,7 +238,7 @@ public class AccordService implements IAccordService, Shutdownable
             instance = NOOP_SERVICE;
             return;
         }
-        AccordService as = new AccordService(AccordTopologyUtils.tcmIdToAccord(tcmId));
+        AccordService as = new AccordService(AccordTopology.tcmIdToAccord(tcmId));
         as.startup();
         instance = as;
     }
@@ -276,6 +278,7 @@ public class AccordService implements IAccordService, Shutdownable
         logger.info("Starting accord with nodeId {}", localId);
         AccordAgent agent = new AccordAgent();
         this.configService = new AccordConfigurationService(localId);
+        this.fastPathCoordinator = AccordFastPathCoordinator.create(localId, configService);
         this.messageSink = new AccordMessageSink(agent, configService);
         this.scheduler = new AccordScheduler();
         this.dataStore = new AccordDataStore();
@@ -310,6 +313,8 @@ public class AccordService implements IAccordService, Shutdownable
         journal.start();
         configService.start();
         ClusterMetadataService.instance().log().addListener(configService);
+        fastPathCoordinator.start();
+        ClusterMetadataService.instance().log().addListener(fastPathCoordinator);
     }
 
     @Override
@@ -666,17 +671,18 @@ public class AccordService implements IAccordService, Shutdownable
         return configService;
     }
 
-    public boolean isAccordManagedKeyspace(String keyspace)
+    @Override
+    public boolean isAccordManagedTable(TableId tableId)
     {
-        return ClusterMetadata.current().accordKeyspaces.contains(keyspace);
+        return ClusterMetadata.current().accordTables.contains(tableId);
     }
 
     @Override
-    public void ensureKeyspaceIsAccordManaged(String keyspace)
+    public void ensureTableIsAccordManaged(TableId tableId)
     {
-        if (isAccordManagedKeyspace(keyspace))
+        if (isAccordManagedTable(tableId))
             return;
-        ClusterMetadataService.instance().commit(new AddAccordKeyspace(keyspace),
+        ClusterMetadataService.instance().commit(new AddAccordTable(tableId),
                                                  metadata -> null,
                                                  (code, message) -> {
                                                      Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS,

src/java/org/apache/cassandra/service/accord/AccordTopology.java

@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import accord.primitives.Ranges;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+
+import accord.local.Node;
+import accord.topology.Shard;
+import accord.topology.Topology;
+import accord.utils.Invariants;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.*;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
+import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
+import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
+
+/**
+ * Deterministically computes accord topology from a ClusterMetadata instance
+ */
+public class AccordTopology
+{
+    public static Node.Id tcmIdToAccord(NodeId nodeId)
+    {
+        return new Node.Id(nodeId.id());
+    }
+
+    private static class ShardLookup extends HashMap<accord.primitives.Range, Shard>
+    {
+        private Shard createOrReuse(accord.primitives.Range range, List<Node.Id> nodes, Set<Node.Id> fastPathElectorate, Set<Node.Id> joining)
+        {
+            Shard prev = get(range);
+            if (prev != null
+                && Objects.equals(prev.nodes, nodes)
+                && Objects.equals(prev.fastPathElectorate, fastPathElectorate)
+                && Objects.equals(prev.joining, joining))
+                return prev;
+
+            return new Shard(range, nodes, fastPathElectorate, joining);
+        }
+    }
+
+    static class KeyspaceShard
+    {
+        private final KeyspaceMetadata keyspace;
+        private final Range<Token> range;
+        private final List<Node.Id> nodes;
+        private final Set<Node.Id> pending;
+
+        private KeyspaceShard(KeyspaceMetadata keyspace, Range<Token> range, List<Node.Id> nodes, Set<Node.Id> pending)
+        {
+            this.keyspace = keyspace;
+            this.range = range;
+            this.nodes = nodes;
+            this.pending = pending;
+        }
+
+        // return the keyspace fast path strategy if the inherit keyspace strategy is used
+        private FastPathStrategy strategyFor(TableMetadata metadata)
+        {
+            FastPathStrategy tableStrategy = metadata.params.fastPath;
+            FastPathStrategy strategy = tableStrategy.kind() != FastPathStrategy.Kind.INHERIT_KEYSPACE
+                                        ? tableStrategy : keyspace.params.fastPath;
+            Invariants.checkState(strategy.kind() != FastPathStrategy.Kind.INHERIT_KEYSPACE);;
+            return strategy;
+        }
+
+        Shard createForTable(TableMetadata metadata, Set<Node.Id> unavailable, Map<Node.Id, String> dcMap, ShardLookup lookup)
+        {
+            TokenRange tokenRange = AccordTopology.range(metadata.id, range);
+
+            Set<Node.Id> fastPath = strategyFor(metadata).calculateFastPath(nodes, unavailable, dcMap);
+
+            return lookup.createOrReuse(tokenRange, nodes, fastPath, pending);
+        }
+
+        private static KeyspaceShard forRange(KeyspaceMetadata keyspace, Range<Token> range, Directory directory, VersionedEndpoints.ForRange reads, VersionedEndpoints.ForRange writes)
+        {
+            // TCM doesn't create wrap around ranges
+            Invariants.checkArgument(!range.isWrapAround() || range.right.equals(range.right.minValue()),
+                                     "wrap around range %s found", range);
+
+            Set<InetAddressAndPort> readEndpoints = reads.endpoints();
+            Set<InetAddressAndPort> writeEndpoints = writes.endpoints();
+            Sets.SetView<InetAddressAndPort> readOnly = Sets.difference(readEndpoints, writeEndpoints);
+            Invariants.checkState(readOnly.isEmpty(), "Read only replicas detected: %s", readOnly);
+
+            List<Node.Id> nodes = writes.endpoints().stream()
+                                        .map(directory::peerId)
+                                        .map(AccordTopology::tcmIdToAccord)
+                                        .sorted().collect(Collectors.toList());
+
+            Set<Node.Id> pending = readEndpoints.equals(writeEndpoints) ?
+                                   Collections.emptySet() :
+                                   writeEndpoints.stream()
+                                                 .filter(e -> !readEndpoints.contains(e))
+                                                 .map(directory::peerId)
+                                                 .map(AccordTopology::tcmIdToAccord)
+                                                 .collect(Collectors.toSet());
+
+            return new KeyspaceShard(keyspace, range, nodes, pending);
+        }
+
+        public static List<KeyspaceShard> forKeyspace(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory, ShardLookup lookup)
+        {
+            ReplicationParams replication = keyspace.params.replication;
+            DataPlacement placement = placements.get(replication);
+
+            List<Range<Token>> ranges = placement.reads.ranges();
+            List<KeyspaceShard> shards = new ArrayList<>(ranges.size());
+            for (Range<Token> range : ranges)
+            {
+                VersionedEndpoints.ForRange reads = placement.reads.forRange(range);
+                VersionedEndpoints.ForRange writes = placement.writes.forRange(range);
+                shards.add(forRange(keyspace, range, directory, reads, writes));
+            }
+            return shards;
+        }
+    }
+
+    static TokenRange minRange(TableId table, Token token)
+    {
+        return new TokenRange(SentinelKey.min(table), new TokenKey(table, token));
+    }
+
+    static TokenRange maxRange(TableId table, Token token)
+    {
+        return new TokenRange(new TokenKey(table, token), SentinelKey.max(table));
+    }
+
+    static TokenRange fullRange(TableId table)
+    {
+        return new TokenRange(SentinelKey.min(table), SentinelKey.max(table));
+    }
+
+    static TokenRange range(TableId table, Range<Token> range)
+    {
+        Token minToken = range.left.minValue();
+        return new TokenRange(range.left.equals(minToken) ? SentinelKey.min(table) : new TokenKey(table, range.left),
+                              range.right.equals(minToken) ? SentinelKey.max(table) : new TokenKey(table, range.right));
+    }
+
+    public static accord.primitives.Ranges toAccordRanges(TableId tableId, Collection<Range<Token>> ranges)
+    {
+        List<Range<Token>> normalizedRanges = Range.normalize(ranges);
+        TokenRange[] tokenRanges = new TokenRange[normalizedRanges.size()];
+        for (int i = 0; i < normalizedRanges.size(); i++)
+            tokenRanges[i] = range(tableId, normalizedRanges.get(i));
+        return Ranges.of(tokenRanges);
+    }
+
+    public static accord.primitives.Ranges toAccordRanges(String keyspace, Collection<Range<Token>> ranges)
+    {
+        Keyspace ks = Keyspace.open(keyspace);
+        Ranges accordRanges = Ranges.EMPTY;
+        if (ks == null)
+            return accordRanges;
+
+        for (TableMetadata tbm : ks.getMetadata().tables)
+        {
+            accordRanges = accordRanges.with(toAccordRanges(tbm.id, ranges));
+        }
+
+        return accordRanges;
+    }
+
+    private static Map<Node.Id, String> createDCMap(Directory directory)
+    {
+        ImmutableMap.Builder<Node.Id, String> builder = ImmutableMap.builder();
+        directory.knownDatacenters().forEach(dc -> {
+            Set<InetAddressAndPort> dcEndpoints = directory.datacenterEndpoints(dc);
+            // nodes aren't added to the endpointsToDCMap until they've joined
+            if (dcEndpoints == null)
+                return;
+            dcEndpoints.forEach(ep -> {
+                NodeId tid = directory.peerId(ep);
+                Node.Id aid = tcmIdToAccord(tid);
+                builder.put(aid, dc);
+            });
+        });
+        return builder.build();
+    }
+
+    public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, AccordFastPath accordFastPath, Predicate<TableId> tablePredicate, ShardLookup lookup)
+    {
+        List<Shard> shards = new ArrayList<>();
+        Set<Node.Id> unavailable = accordFastPath.unavailableIds();
+        Map<Node.Id, String> dcMap = createDCMap(directory);
+
+        for (KeyspaceMetadata keyspace : schema.getKeyspaces())
+        {
+            List<TableMetadata> tables = keyspace.tables.stream().filter(tbl -> tablePredicate.test(tbl.id)).collect(Collectors.toList());
+            if (tables.isEmpty())
+                continue;
+            List<KeyspaceShard> ksShards = KeyspaceShard.forKeyspace(keyspace, placements, directory, lookup);
+            tables.forEach(table -> ksShards.forEach(shard -> shards.add(shard.createForTable(table, unavailable, dcMap, lookup))));
+        }
+
+        shards.sort((a, b) -> a.range.compare(b.range));
+        return new Topology(epoch.getEpoch(), shards.toArray(new Shard[0]));
+    }
+
+    public static Topology createAccordTopology(ClusterMetadata metadata, Predicate<TableId> tablePredicate, ShardLookup lookup)
+    {
+        return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, tablePredicate, lookup);
+    }
+
+    public static Topology createAccordTopology(ClusterMetadata metadata, Predicate<TableId> tablePredicate)
+    {
+        return createAccordTopology(metadata, tablePredicate, new ShardLookup());
+    }
+
+    public static Topology createAccordTopology(ClusterMetadata metadata, Topology current)
+    {
+        return createAccordTopology(metadata, metadata.accordTables::contains, createShardLookup(current));
+    }
+
+    public static Topology createAccordTopology(ClusterMetadata metadata)
+    {
+        return createAccordTopology(metadata, (Topology) null);
+    }
+
+    public static EndpointMapping directoryToMapping(EndpointMapping mapping, long epoch, Directory directory)
+    {
+        EndpointMapping.Builder builder = EndpointMapping.builder(epoch);
+        for (NodeId id : directory.peerIds())
+            builder.add(directory.endpoint(id), tcmIdToAccord(id));
+
+        // There are cases where nodes are removed from the cluster (host replacement, decom, etc.), but inflight events may still be happening;
+        // keep the ids around so pending events do not fail with a mapping error
+        for (Node.Id id : mapping.differenceIds(builder))
+            builder.add(mapping.mappedEndpoint(id), id);
+        return builder.build();
+    }
+
+    private static ShardLookup createShardLookup(Topology topology)
+    {
+        ShardLookup map = new ShardLookup();
+
+        if (topology == null)
+            return map;
+
+        topology.forEach(shard -> map.put(shard.range, shard));
+        return map;
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java

@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.accord;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Sets;
-
-import accord.local.Node;
-import accord.primitives.Ranges;
-import accord.topology.Shard;
-import accord.topology.Topology;
-import accord.utils.Invariants;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.EndpointsForRange;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.ReplicationParams;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
-import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
-import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.tcm.membership.Directory;
-import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.tcm.ownership.DataPlacements;
-
-public class AccordTopologyUtils
-{
-    public static Node.Id tcmIdToAccord(NodeId nodeId)
-    {
-        return new Node.Id(nodeId.id());
-    }
-
-    private static Shard createShard(TokenRange range, Directory directory, EndpointsForRange reads, EndpointsForRange writes)
-    {
-        Function<InetAddressAndPort, Node.Id> endpointMapper = e -> {
-            NodeId tcmId = directory.peerId(e);
-            return tcmIdToAccord(tcmId);
-        };
-        Set<InetAddressAndPort> endpoints = reads.endpoints();
-        Set<InetAddressAndPort> writeEndpoints = writes.endpoints();
-        List<Node.Id> nodes = endpoints.stream().map(endpointMapper).sorted().collect(Collectors.toList());
-        Set<Node.Id> fastPath = new HashSet<>(nodes);  // TODO: support fast path updates
-        Set<Node.Id> pending = endpoints.equals(writeEndpoints) ?
-                               Collections.emptySet() :
-                               writeEndpoints.stream().filter(e -> !endpoints.contains(e)).map(endpointMapper).collect(Collectors.toSet());
-
-        Sets.SetView<InetAddressAndPort> readOnly = Sets.difference(endpoints, writeEndpoints);
-        Invariants.checkState(readOnly.isEmpty(), "Read only replicas detected: %s", readOnly);
-        return new Shard(range, nodes, fastPath, pending);
-    }
-
-    static TokenRange minRange(String keyspace, Token token)
-    {
-        return new TokenRange(SentinelKey.min(keyspace), new TokenKey(keyspace, token));
-    }
-
-    static TokenRange maxRange(String keyspace, Token token)
-    {
-        return new TokenRange(new TokenKey(keyspace, token), SentinelKey.max(keyspace));
-    }
-
-    static TokenRange fullRange(String keyspace)
-    {
-        return new TokenRange(SentinelKey.min(keyspace), SentinelKey.max(keyspace));
-    }
-
-    static TokenRange range(String keyspace, Range<Token> range)
-    {
-        Token minToken = range.left.minValue();
-        return new TokenRange(range.left.equals(minToken) ? SentinelKey.min(keyspace) : new TokenKey(keyspace, range.left),
-                              range.right.equals(minToken) ? SentinelKey.max(keyspace) : new TokenKey(keyspace, range.right));
-    }
-
-    public static accord.primitives.Ranges toAccordRanges(String keyspace, Collection<Range<Token>> ranges)
-    {
-        List<Range<Token>> normalizedRanges = Range.normalize(ranges);
-        TokenRange[] tokenRanges = new TokenRange[normalizedRanges.size()];
-        for (int i = 0; i < normalizedRanges.size(); i++)
-            tokenRanges[i] = range(keyspace, normalizedRanges.get(i));
-        return Ranges.of(tokenRanges);
-    }
-
-    public static List<Shard> createShards(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory)
-    {
-        ReplicationParams replication = keyspace.params.replication;
-        DataPlacement placement = placements.get(replication);
-
-        List<Range<Token>> ranges = placement.reads.ranges();
-        List<Shard> shards = new ArrayList<>(ranges.size());
-        for (Range<Token> range : ranges)
-        {
-            // TODO (consider, low priority): flesh out how Accord and Transient Replicas work together
-            // Accord needs to be able to read the full data from a single replica, but with transient ones they may only have a hash.
-            EndpointsForRange reads = placement.reads.forRange(range).get().filter(r -> r.isFull());
-            EndpointsForRange writes = placement.writes.forRange(range).get().filter(r -> r.isFull());
-
-            // TCM doesn't create wrap around ranges
-            Invariants.checkArgument(!range.isWrapAround() || range.right.equals(range.right.minValue()),
-                                     "wrap around range %s found", range);
-            shards.add(createShard(range(keyspace.name, range), directory, reads, writes));
-        }
-
-        return shards;
-    }
-
-    public static Topology createAccordTopology(ClusterMetadata cm, Predicate<String> keyspacePredicate)
-    {
-        List<Shard> shards = new ArrayList<>();
-        for (KeyspaceMetadata keyspace : cm.schema.getKeyspaces())
-        {
-            if (!keyspacePredicate.test(keyspace.name))
-                continue;
-            shards.addAll(createShards(keyspace, cm.placements, cm.directory));
-        }
-        shards.sort((a, b) -> a.range.compare(b.range));
-        return new Topology(cm.epoch.getEpoch(), shards.toArray(new Shard[0]));
-    }
-
-    public static EndpointMapping directoryToMapping(EndpointMapping mapping, long epoch, Directory directory)
-    {
-        EndpointMapping.Builder builder = EndpointMapping.builder(epoch);
-        for (NodeId id : directory.peerIds())
-            builder.add(directory.endpoint(id), tcmIdToAccord(id));
-
-        // There are cases where nodes are removed from the cluster (host replacement, decom, etc.), but inflight events may still be happening;
-        // keep the ids around so pending events do not fail with a mapping error
-        for (Node.Id id : mapping.differenceIds(builder))
-            builder.add(mapping.mappedEndpoint(id), id);
-        return builder.build();
-    }
-
-    public static Topology createAccordTopology(ClusterMetadata metadata)
-    {
-        return createAccordTopology(metadata, metadata.accordKeyspaces::contains);
-    }
-}

src/java/org/apache/cassandra/service/accord/CommandsForRanges.java

@@ -59,6 +59,7 @@ import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
@@ -407,13 +408,13 @@ public class CommandsForRanges
 
     public Iterable<DomainInfo> search(AbstractKeys<Key> keys)
     {
-        // group by the keyspace, as ranges are based off TokenKey, which is scoped to a range
-        Map<String, List<Key>> groupByKeyspace = new TreeMap<>();
+        // group by the table, as ranges are based off TokenKey, which is scoped to a range
+        Map<TableId, List<Key>> groupByTable = new TreeMap<>();
         for (Key key : keys)
-            groupByKeyspace.computeIfAbsent(((PartitionKey) key).keyspace(), ignore -> new ArrayList<>()).add(key);
+            groupByTable.computeIfAbsent(((PartitionKey) key).table(), ignore -> new ArrayList<>()).add(key);
         return () -> new AbstractIterator<DomainInfo>()
         {
-            Iterator<String> ksIt = groupByKeyspace.keySet().iterator();
+            Iterator<TableId> tblIt = groupByTable.keySet().iterator();
             Iterator<Map.Entry<Range, Set<RangeCommandSummary>>> rangeIt;
 
             @Override
@@ -427,13 +428,13 @@ public class CommandsForRanges
                         return result(next.getKey(), next.getValue());
                     }
                     rangeIt = null;
-                    if (!ksIt.hasNext())
+                    if (!tblIt.hasNext())
                     {
-                        ksIt = null;
+                        tblIt = null;
                         return endOfData();
                     }
-                    String ks = ksIt.next();
-                    List<Key> keys = groupByKeyspace.get(ks);
+                    TableId tbl = tblIt.next();
+                    List<Key> keys = groupByTable.get(tbl);
                     Map<Range, Set<RangeCommandSummary>> groupByRange = new TreeMap<>(Range::compare);
                     for (Key key : keys)
                     {

src/java/org/apache/cassandra/service/accord/EndpointMapping.java

@@ -58,13 +58,13 @@ class EndpointMapping implements AccordEndpointMapper
     }
 
     @Override
-    public Node.Id mappedId(InetAddressAndPort endpoint)
+    public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
     {
         return mapping.inverse().get(endpoint);
     }
 
     @Override
-    public InetAddressAndPort mappedEndpoint(Node.Id id)
+    public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
     {
         return mapping.get(id);
     }

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -40,16 +40,23 @@ import accord.topology.TopologyManager;
 import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.accord.api.AccordRoutableKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordScheduler;
 import org.apache.cassandra.service.accord.txn.TxnResult;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.tcm.transformations.AddAccordTable;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
 
@@ -72,7 +79,7 @@ public interface IAccordService
         String ks = cfs.keyspace.getName();
         Ranges accordRanges = Ranges.of(ranges
              .stream()
-             .map(r -> new TokenRange(new TokenKey(ks, r.left), new TokenKey(ks, r.right)))
+             .map(r -> new TokenRange(new TokenKey(cfs.getTableId(), r.left), new TokenKey(cfs.getTableId(), r.right)))
              .collect(Collectors.toList())
              .toArray(new accord.primitives.Range[0]));
         try
@@ -109,10 +116,10 @@ public interface IAccordService
 
     /**
      * Temporary method to avoid double-streaming keyspaces
-     * @param keyspace
+     * @param tableId
      * @return
      */
-    boolean isAccordManagedKeyspace(String keyspace);
+    boolean isAccordManagedTable(TableId tableId);
 
     /**
      * Fetch the redundnant befores for every command store
@@ -121,23 +128,45 @@ public interface IAccordService
 
     default Id nodeId() { throw new UnsupportedOperationException(); }
 
-    default void maybeConvertKeyspacesToAccord(Txn txn)
+    default void maybeConvertTablesToAccord(Txn txn)
     {
-        Set<String> allKeyspaces = new HashSet<>();
-        txn.keys().forEach(key -> allKeyspaces.add(((AccordRoutableKey) key).keyspace()));
+        Set<TableId> allTables = new HashSet<>();
+        Set<TableId> newTables = new HashSet<>();
+        txn.keys().forEach(key -> {
+            TableId table = ((AccordRoutableKey) key).table();
+            if (allTables.add(table) && !isAccordManagedTable(table))
+                newTables.add(table);
+        });
 
-        for (String keyspace : allKeyspaces)
+        if (newTables.isEmpty())
+            return;
+
+        for (TableId table : newTables)
+            AddAccordTable.addTable(table);
+
+        // we need to avoid creating a txnId in an epoch when no one has any ranges
+        FBUtilities.waitOnFuture(epochReady(ClusterMetadata.current().epoch));
+
+        for (TableId table : allTables)
         {
-
-            ensureKeyspaceIsAccordManaged(keyspace);
-        }
-
-        for (String keyspace : allKeyspaces)
-        {
-            if (!AccordService.instance().isAccordManagedKeyspace(keyspace))
-                throw new IllegalStateException(keyspace + " is not an accord managed keyspace");
+            if (!isAccordManagedTable(table))
+                throw new IllegalStateException(table + " is not an accord managed table");
         }
     }
 
-    void ensureKeyspaceIsAccordManaged(String keyspace);
+    void ensureTableIsAccordManaged(TableId tableId);
+
+    default void ensureTableIsAccordManaged(String keyspace, String table)
+    {
+        // TODO: remove when accord enabled is handled via schema
+        TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
+        ensureTableIsAccordManaged(metadata.id);
+    }
+
+    default void ensureKeyspaceIsAccordManaged(String keyspace)
+    {
+        // TODO: remove when accord enabled is handled via schema
+        Keyspace ks = Keyspace.open(keyspace);
+        ks.getMetadata().tables.forEach(metadata -> ensureTableIsAccordManaged(metadata.id));
+    }
 }

src/java/org/apache/cassandra/service/accord/TokenRange.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
 
@@ -40,25 +41,25 @@ public class TokenRange extends Range.EndInclusive
     public TokenRange(AccordRoutingKey start, AccordRoutingKey end)
     {
         super(start, end);
-        Invariants.checkArgument(start.keyspace().equals(end.keyspace()),
+        Invariants.checkArgument(start.table().equals(end.table()),
                                  "Token ranges cannot cover more than one keyspace start:%s, end:%s",
                                  start, end);
     }
 
-    public String keyspace()
+    public TableId table()
     {
-        return ((AccordRoutingKey) start()).keyspace();
+        return ((AccordRoutingKey) start()).table();
     }
 
     @VisibleForTesting
-    public Range withKeyspace(String ks)
+    public Range withTable(TableId table)
     {
-        return new TokenRange(((AccordRoutingKey) start()).withKeyspace(ks), ((AccordRoutingKey) end()).withKeyspace(ks));
+        return new TokenRange(((AccordRoutingKey) start()).withTable(table), ((AccordRoutingKey) end()).withTable(table));
     }
 
-    public static TokenRange fullRange(String keyspace)
+    public static TokenRange fullRange(TableId table)
     {
-        return new TokenRange(SentinelKey.min(keyspace), SentinelKey.max(keyspace));
+        return new TokenRange(SentinelKey.min(table), SentinelKey.max(table));
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/api/AccordAgent.java

@@ -103,7 +103,7 @@ public class AccordAgent implements Agent
     @Override
     public void onUncaughtException(Throwable t)
     {
-        // TODO: this
+        logger.error("Uncaught accord exception", t);
         JVMStabilityInspector.uncaughtException(Thread.currentThread(), t);
     }
 

src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java

@@ -22,25 +22,30 @@ import java.util.Objects;
 
 import accord.primitives.RoutableKey;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 
 public abstract class AccordRoutableKey implements RoutableKey
 {
-    final String keyspace; // TODO (desired): use an id (TrM)
+    final TableId table; // TODO (desired): use an id (TrM)
 
-    protected AccordRoutableKey(String keyspace)
+    protected AccordRoutableKey(TableId table)
     {
-        this.keyspace = keyspace;
+        this.table = table;
+    }
+
+    public TableId table()
+    {
+        return table;
     }
 
-    public final String keyspace() { return keyspace; }
     public abstract Token token();
 
     @Override
     public Object prefix()
     {
-        return keyspace;
+        return table;
     }
 
     @Override
@@ -52,7 +57,7 @@ public abstract class AccordRoutableKey implements RoutableKey
     @Override
     public int hashCode()
     {
-        return Objects.hash(keyspace, token().tokenHash());
+        return Objects.hash(table, token().tokenHash());
     }
 
     @Override
@@ -63,7 +68,7 @@ public abstract class AccordRoutableKey implements RoutableKey
 
     public final int compareTo(AccordRoutableKey that)
     {
-        int cmp = this.keyspace().compareTo(that.keyspace());
+        int cmp = this.table().compareTo(that.table());
         if (cmp != 0)
             return cmp;
 
@@ -80,7 +85,7 @@ public abstract class AccordRoutableKey implements RoutableKey
 
         if (this.getClass() == TokenKey.class)
             return that.getClass() == TokenKey.class ? 0 : 1;
-        return that.getClass() == TokenKey.class ? -1 : ((PartitionKey)this).tableId.compareTo(((PartitionKey)that).tableId);
+        return that.getClass() == TokenKey.class ? -1 : 0;
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.io.util.DataInputBuffer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
@@ -53,14 +54,14 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         TOKEN, SENTINEL
     }
 
-    protected AccordRoutingKey(String keyspace)
+    protected AccordRoutingKey(TableId table)
     {
-        super(keyspace);
+        super(table);
     }
 
     public abstract RoutingKeyKind kindOfRoutingKey();
     public abstract long estimatedSizeOnHeap();
-    public abstract AccordRoutingKey withKeyspace(String ks);
+    public abstract AccordRoutingKey withTable(TableId table);
 
     public SentinelKey asSentinelKey()
     {
@@ -84,16 +85,16 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
 
         private final boolean isMin;
 
-        private SentinelKey(String keyspace, boolean isMin)
+        private SentinelKey(TableId table, boolean isMin)
         {
-            super(keyspace);
+            super(table);
             this.isMin = isMin;
         }
 
         @Override
         public int hashCode()
         {
-            return Objects.hash(keyspace, isMin);
+            return Objects.hash(table, isMin);
         }
 
         @Override
@@ -108,26 +109,25 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
             return EMPTY_SIZE;
         }
 
-        @Override
-        public AccordRoutingKey withKeyspace(String ks)
+        public AccordRoutingKey withTable(TableId table)
         {
-            return new SentinelKey(ks, isMin);
+            return new SentinelKey(table, isMin);
         }
 
-        public static SentinelKey min(String keyspace)
+        public static SentinelKey min(TableId table)
         {
-            return new SentinelKey(keyspace, true);
+            return new SentinelKey(table, true);
         }
 
-        public static SentinelKey max(String keyspace)
+        public static SentinelKey max(TableId table)
         {
-            return new SentinelKey(keyspace, false);
+            return new SentinelKey(table, false);
         }
 
         public TokenKey toTokenKey()
         {
             IPartitioner partitioner = getPartitioner();
-            return new TokenKey(keyspace, isMin ?
+            return new TokenKey(table, isMin ?
                                          partitioner.getMinimumToken().nextValidToken() :
                                          partitioner.getMaximumToken().decreaseSlightly());
         }
@@ -154,22 +154,22 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
             @Override
             public void serialize(SentinelKey key, DataOutputPlus out, int version) throws IOException
             {
+                key.table.serialize(out);
                 out.writeBoolean(key.isMin);
-                out.writeUTF(key.keyspace);
             }
 
             @Override
             public SentinelKey deserialize(DataInputPlus in, int version) throws IOException
             {
+                TableId table = TableId.deserialize(in);
                 boolean isMin = in.readBoolean();
-                String keyspace = in.readUTF();
-                return new SentinelKey(keyspace, isMin);
+                return new SentinelKey(table, isMin);
             }
 
             @Override
             public long serializedSize(SentinelKey key, int version)
             {
-                return TypeSizes.BOOL_SIZE + TypeSizes.sizeof(key.keyspace);
+                return key.table().serializedSize() + TypeSizes.BOOL_SIZE;
             }
         };
 
@@ -189,8 +189,8 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         public Range asRange()
         {
             AccordRoutingKey before = token.isMinimum()
-                                      ? new SentinelKey(keyspace, true)
-                                      : new TokenKey(keyspace, token.decreaseSlightly());
+                                      ? new SentinelKey(table, true)
+                                      : new TokenKey(table, token.decreaseSlightly());
 
             return new TokenRange(before, this);
         }
@@ -202,15 +202,15 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         }
 
         final Token token;
-        public TokenKey(String keyspace, Token token)
+        public TokenKey(TableId tableId, Token token)
         {
-            super(keyspace);
+            super(tableId);
             this.token = token;
         }
 
         public TokenKey withToken(Token token)
         {
-            return new TokenKey(keyspace, token);
+            return new TokenKey(table, token);
         }
 
         @Override
@@ -236,10 +236,9 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
             return EMPTY_SIZE + token().getHeapSize();
         }
 
-        @Override
-        public AccordRoutingKey withKeyspace(String ks)
+        public AccordRoutingKey withTable(TableId table)
         {
-            return new TokenKey(ks, token);
+            return new TokenKey(table, token);
         }
 
         public static final Serializer serializer = new Serializer();
@@ -250,22 +249,22 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
             @Override
             public void serialize(TokenKey key, DataOutputPlus out, int version) throws IOException
             {
-                out.writeUTF(key.keyspace);
+                key.table.serialize(out);
                 Token.compactSerializer.serialize(key.token, out, version);
             }
 
             @Override
             public TokenKey deserialize(DataInputPlus in, int version) throws IOException
             {
-                String keyspace = in.readUTF();
+                TableId table = TableId.deserialize(in);
                 Token token = Token.compactSerializer.deserialize(in, getPartitioner(), version);
-                return new TokenKey(keyspace, token);
+                return new TokenKey(table, token);
             }
 
             @Override
             public long serializedSize(TokenKey key, int version)
             {
-                return TypeSizes.sizeof(key.keyspace) + Token.compactSerializer.serializedSize(key.token(), version);
+                return key.table.serializedSize() + Token.compactSerializer.serializedSize(key.token(), version);
             }
         }
     }
@@ -370,15 +369,15 @@ public abstract class AccordRoutingKey extends AccordRoutableKey implements Rout
         @Override
         public List<Ranges> split(Ranges ranges)
         {
-            Map<String, List<Range>> byKeyspace = new TreeMap<>();
+            Map<TableId, List<Range>> byTable = new TreeMap<>();
             for (Range range : ranges)
             {
-                byKeyspace.computeIfAbsent(((AccordRoutableKey)range.start()).keyspace, ignore -> new ArrayList<>())
+                byTable.computeIfAbsent(((AccordRoutableKey)range.start()).table, ignore -> new ArrayList<>())
                           .add(range);
             }
 
             List<Ranges> results = new ArrayList<>();
-            for (List<Range> keyspaceRanges : byKeyspace.values())
+            for (List<Range> keyspaceRanges : byTable.values())
             {
                 List<Ranges> splits = subSplitter.split(Ranges.ofSortedAndDeoverlapped(keyspaceRanges.toArray(new Range[0])));
 

src/java/org/apache/cassandra/service/accord/api/PartitionKey.java

@@ -53,16 +53,14 @@ public final class PartitionKey extends AccordRoutableKey implements Key
     static
     {
         DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.EMPTY_BYTE_BUFFER);
-        EMPTY_SIZE = ObjectSizes.measureDeep(new PartitionKey(null, null, key));
+        EMPTY_SIZE = ObjectSizes.measureDeep(new PartitionKey(null, key));
     }
 
-    final TableId tableId; // TODO (expected): move to PartitionKey
     final DecoratedKey key;
 
-    public PartitionKey(String keyspace, TableId tableId, DecoratedKey key)
+    public PartitionKey(TableId tableId, DecoratedKey key)
     {
-        super(keyspace);
-        this.tableId = tableId;
+        super(tableId);
         this.key = key;
     }
 
@@ -73,21 +71,19 @@ public final class PartitionKey extends AccordRoutableKey implements Key
 
     public static PartitionKey of(PartitionUpdate update)
     {
-        return new PartitionKey(update.metadata().keyspace, update.metadata().id, update.partitionKey());
+        return new PartitionKey(update.metadata().id, update.partitionKey());
     }
 
     public static PartitionKey of(Partition partition)
     {
-        return new PartitionKey(partition.metadata().keyspace, partition.metadata().id, partition.partitionKey());
+        return new PartitionKey(partition.metadata().id, partition.partitionKey());
     }
 
     public static PartitionKey of(SinglePartitionReadCommand command)
     {
-        return new PartitionKey(command.metadata().keyspace, command.metadata().id, command.partitionKey());
+        return new PartitionKey(command.metadata().id, command.partitionKey());
     }
 
-    public final TableId tableId() { return tableId; }
-
     @Override
     public Token token()
     {
@@ -102,7 +98,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
     @Override
     public RoutingKey toUnseekable()
     {
-        return new TokenKey(keyspace, token());
+        return new TokenKey(table, token());
     }
 
     public long estimatedSizeOnHeap()
@@ -131,14 +127,14 @@ public final class PartitionKey extends AccordRoutableKey implements Key
         @Override
         public void serialize(PartitionKey key, DataOutputPlus out, int version) throws IOException
         {
-            key.tableId().serialize(out);
+            key.table().serialize(out);
             ByteBufferUtil.writeWithShortLength(key.partitionKey().getKey(), out);
         }
 
         public <V> int serialize(PartitionKey key, V dst, ValueAccessor<V> accessor, int offset)
         {
             int position = offset;
-            position += key.tableId().serialize(dst, accessor, position);
+            position += key.table().serialize(dst, accessor, position);
             ByteBuffer bytes = key.partitionKey().getKey();
             int numBytes = ByteBufferAccessor.instance.size(bytes);
             Preconditions.checkState(numBytes <= Short.MAX_VALUE);
@@ -154,7 +150,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
             TableId tableId = TableId.deserialize(in);
             TableMetadata metadata = Schema.instance.getExistingTableMetadata(tableId);
             DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in));
-            return new PartitionKey(metadata.keyspace, tableId, key);
+            return new PartitionKey(tableId, key);
         }
 
         public <V> PartitionKey deserialize(V src, ValueAccessor<V> accessor, int offset) throws IOException
@@ -167,7 +163,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
             ByteBuffer bytes = ByteBuffer.allocate(numBytes);
             accessor.copyTo(src, offset, bytes, ByteBufferAccessor.instance, 0, numBytes);
             DecoratedKey key = metadata.partitioner.decorateKey(bytes);
-            return new PartitionKey(metadata.keyspace, tableId, key);
+            return new PartitionKey(tableId, key);
         }
 
         @Override
@@ -178,7 +174,7 @@ public final class PartitionKey extends AccordRoutableKey implements Key
 
         public long serializedSize(PartitionKey key)
         {
-            return key.tableId().serializedSize() + ByteBufferUtil.serializedSizeWithShortLength(key.partitionKey().getKey());
+            return key.table().serializedSize() + ByteBufferUtil.serializedSizeWithShortLength(key.partitionKey().getKey());
         }
     }
 }

src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java

@@ -125,10 +125,17 @@ public class AsyncLoader
                                                  List<AsyncChain<?>> listenChains)
     {
         referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains);
-        if (keyHistory == KeyHistory.DEPS)
-            referenceAndAssembleReadsForKey(key, context.depsCommandsForKeys, commandStore.depsCommandsForKeyCache(), listenChains);
-        if (keyHistory == KeyHistory.ALL)
-            referenceAndAssembleReadsForKey(key, context.allCommandsForKeys, commandStore.allCommandsForKeyCache(), listenChains);
+        // recovery operations also need the deps data for their preaccept logic
+        switch (keyHistory)
+        {
+            case ALL:
+                referenceAndAssembleReadsForKey(key, context.allCommandsForKeys, commandStore.allCommandsForKeyCache(), listenChains);
+            case DEPS:
+                referenceAndAssembleReadsForKey(key, context.depsCommandsForKeys, commandStore.depsCommandsForKeyCache(), listenChains);
+            case NONE:
+                break;
+            default: throw new IllegalArgumentException("Unhandled keyhistory: " + keyHistory);
+        }
         referenceAndAssembleReadsForKey(key, context.updatesForKeys, commandStore.updatesForKeyCache(), listenChains);
     }
 

src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java

@@ -210,6 +210,7 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
 
     private void fail(Throwable throwable)
     {
+        commandStore.agent().onUncaughtException(throwable);
         commandStore.checkInStoreThread();
         Invariants.nonNull(throwable);
 

src/java/org/apache/cassandra/service/accord/fastpath/FastPathStrategy.java

@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.fastpath;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableMap;
+
+import accord.local.Node;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.utils.LocalizeString;
+
+public interface FastPathStrategy
+{
+    enum Kind
+    {
+        SIMPLE, PARAMETERIZED, INHERIT_KEYSPACE;
+
+        static final String KEY = "kind";
+        private static final Map<String, Kind> LOOKUP;
+        static
+        {
+            ImmutableMap.Builder<String, Kind> builder = ImmutableMap.builder();
+            builder.put(SIMPLE.name(), SIMPLE);
+            builder.put(PARAMETERIZED.name(), PARAMETERIZED);
+            builder.put(INHERIT_KEYSPACE.name(), INHERIT_KEYSPACE);
+            LOOKUP = builder.build();
+        }
+
+        public byte asByte()
+        {
+            return (byte) ordinal();
+        }
+
+        public static Kind fromByte(byte i)
+        {
+            return values()[i];
+        }
+
+        @Nullable
+        public static Kind fromString(String s)
+        {
+            return LOOKUP.get(LocalizeString.toUpperCaseLocalized(s));
+        }
+
+        @Nullable
+        private static Kind fromMap(Map<String, String> map)
+        {
+            String name = map.remove(KEY);
+            return name != null ? fromString(name) : null;
+        }
+    }
+
+    /**
+     * @param nodes expected to be sorted deterministically
+     * @param unavailable
+     * @param dcMap
+     * @return
+     */
+    Set<Node.Id> calculateFastPath(List<Node.Id> nodes, Set<Node.Id> unavailable, Map<Node.Id, String> dcMap);
+
+    Kind kind();
+
+    Map<String, String> asMap();
+
+    String asCQL();
+
+    static FastPathStrategy fromMap(Map<String, String> map)
+    {
+        if (map == null || map.isEmpty())
+            return SimpleFastPathStrategy.instance;
+
+        map = new HashMap<>(map);
+        Kind kind = Kind.fromMap(map);
+        if (kind == null)
+            return map.isEmpty()
+                   ? simple()
+                   : ParameterizedFastPathStrategy.fromMap(map);
+
+        switch (kind)
+        {
+            case SIMPLE:
+                return simple();
+            case PARAMETERIZED:
+                return ParameterizedFastPathStrategy.fromMap(map);
+            case INHERIT_KEYSPACE:
+                return inheritKeyspace();
+            default:
+                throw new IllegalArgumentException("Unhandled strategy kind: " + kind);
+        }
+    }
+
+    static FastPathStrategy tableStrategyFromString(String s)
+    {
+        s = LocalizeString.toLowerCaseLocalized(s).trim();
+        if (s.equals("keyspace"))
+            return InheritKeyspaceFastPathStrategy.instance;
+        if (s.equals("simple"))
+            return SimpleFastPathStrategy.instance;
+
+        throw new ConfigurationException("Fast path strategy must either be 'keyspace', `default` or a map size and optional dcs {'size':n, 'dcs': dc0,dc1...");
+    }
+
+    static FastPathStrategy keyspaceStrategyFromString(String s)
+    {
+        s = LocalizeString.toLowerCaseLocalized(s).trim();
+        if (s.equals("simple"))
+            return SimpleFastPathStrategy.instance;
+
+        throw new ConfigurationException("Fast path strategy must either be `default` or a map size and optional dcs {'size':n, 'dcs': dc0,dc1...");
+    }
+
+    static FastPathStrategy simple()
+    {
+        return SimpleFastPathStrategy.instance;
+    }
+
+    static FastPathStrategy inheritKeyspace()
+    {
+        return InheritKeyspaceFastPathStrategy.instance;
+    }
+
+    MetadataSerializer<FastPathStrategy> serializer = new MetadataSerializer<FastPathStrategy>()
+    {
+        public void serialize(FastPathStrategy strategy, DataOutputPlus out, Version version) throws IOException
+        {
+            Kind type = strategy.kind();
+            out.write(type.asByte());
+            if (type == Kind.PARAMETERIZED)
+                ParameterizedFastPathStrategy.serializer.serialize((ParameterizedFastPathStrategy) strategy, out, version);
+        }
+
+        public FastPathStrategy deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            Kind type =  Kind.fromByte(in.readByte());
+            switch (type)
+            {
+                case SIMPLE:
+                    return simple();
+                case PARAMETERIZED:
+                    return ParameterizedFastPathStrategy.serializer.deserialize(in, version);
+                case INHERIT_KEYSPACE:
+                    return inheritKeyspace();
+                default:
+                    throw new IllegalArgumentException("Unhandled type: " + type);
+            }
+        }
+
+        public long serializedSize(FastPathStrategy strategy, Version version)
+        {
+            long size = TypeSizes.BYTE_SIZE;
+            if (strategy.kind() == Kind.PARAMETERIZED)
+                size += ParameterizedFastPathStrategy.serializer.serializedSize((ParameterizedFastPathStrategy) strategy, version);
+            return size;
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/fastpath/InheritKeyspaceFastPathStrategy.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.fastpath;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+
+import accord.local.Node;
+
+public class InheritKeyspaceFastPathStrategy implements FastPathStrategy
+{
+    static final FastPathStrategy instance = new InheritKeyspaceFastPathStrategy();
+
+    private static final Map<String, String> SCHEMA_PARAMS = ImmutableMap.of(Kind.KEY, Kind.INHERIT_KEYSPACE.name());
+
+    private InheritKeyspaceFastPathStrategy() {}
+
+    @Override
+    public Set<Node.Id> calculateFastPath(List<Node.Id> nodes, Set<Node.Id> unavailable, Map<Node.Id, String> dcMap)
+    {
+        throw new IllegalStateException("InheritKeyspaceFastPathStrategy should be replaced before calculateFastPath is called");
+    }
+
+    @Override
+    public Kind kind()
+    {
+        return Kind.INHERIT_KEYSPACE;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "keyspace";
+    }
+
+    public Map<String, String> asMap()
+    {
+        return SCHEMA_PARAMS;
+    }
+
+    @Override
+    public String asCQL()
+    {
+        return "'keyspace'";
+    }
+}

src/java/org/apache/cassandra/service/accord/fastpath/ParameterizedFastPathStrategy.java

@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.fastpath;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import accord.api.VisibleForImplementation;
+import accord.local.Node;
+import accord.topology.Shard;
+import accord.utils.Invariants;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+import javax.annotation.Nonnull;
+
+public class ParameterizedFastPathStrategy implements FastPathStrategy
+{
+    static final String SIZE = "size";
+    static final String DCS = "dcs";
+    private static final Joiner DC_JOINER = Joiner.on(',');
+    private static final Pattern COMMA_SEPARATOR = Pattern.compile(",");
+    private static final Pattern COLON_SEPARATOR = Pattern.compile(":");
+
+    static class WeightedDc implements Comparable<WeightedDc>
+    {
+        private static final WeightedDc UNSPECIFIED = new WeightedDc("<unspecified>", Integer.MAX_VALUE, true);
+        private static final MetadataSerializer<WeightedDc> serializer = new MetadataSerializer<WeightedDc>()
+        {
+            public void serialize(WeightedDc dc, DataOutputPlus out, Version version) throws IOException
+            {
+                out.writeUTF(dc.name);
+                out.writeUnsignedVInt(dc.weight);
+                out.writeBoolean(dc.autoWeight);
+            }
+
+            public WeightedDc deserialize(DataInputPlus in, Version version) throws IOException
+            {
+                return new WeightedDc(in.readUTF(),
+                                      in.readUnsignedVInt32(),
+                                      in.readBoolean());
+            }
+
+            public long serializedSize(WeightedDc dc, Version version)
+            {
+                return TypeSizes.sizeof(dc.name) + TypeSizes.sizeofUnsignedVInt(dc.weight) + TypeSizes.BOOL_SIZE;
+            }
+        };
+
+        final String name;
+        final int weight;
+        final boolean autoWeight;
+
+        public WeightedDc(String name, int weight, boolean autoWeight)
+        {
+            this.name = name;
+            this.weight = weight;
+            this.autoWeight = autoWeight;
+        }
+
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            WeightedDc that = (WeightedDc) o;
+            return weight == that.weight && autoWeight == that.autoWeight && Objects.equals(name, that.name);
+        }
+
+        public int hashCode()
+        {
+            return Objects.hash(name, weight, autoWeight);
+        }
+
+        @Override
+        public int compareTo(@Nonnull WeightedDc that)
+        {
+            int cmp = Integer.compare(this.weight, that.weight);
+            if (cmp != 0) return cmp;
+            return this.name.compareTo(that.name);
+        }
+
+        public String toString()
+        {
+            return autoWeight ? name : name + ':' + weight;
+        }
+
+        static String validateDC(String dc)
+        {
+            dc = dc.trim();
+            if (dc.isEmpty())
+                throw cfe("dc name must not be empty", DCS);
+            return dc;
+        }
+
+        static int validateWeight(String w)
+        {
+            int weight = Integer.parseInt(w);
+            if (weight < 0)
+                throw cfe("DC weights must be zero or positive");
+            return weight;
+        }
+
+        static WeightedDc fromString(String s, int idx)
+        {
+            s = s.trim();
+            if (s.isEmpty())
+                throw cfe("%s entries must not be empty", DCS);
+
+            String[] parts = COLON_SEPARATOR.split(s);
+            if (parts.length == 1)
+                return new WeightedDc(validateDC(parts[0]), idx, true);
+            else if (parts.length == 2)
+                return new WeightedDc(validateDC(parts[0]), validateWeight(parts[1]), false);
+            else
+                throw cfe("Invalid dc weighting syntax %s, use <dc>:<weight>");
+        }
+    }
+
+    public final int size;
+    private final ImmutableMap<String, WeightedDc> dcs;
+
+    ParameterizedFastPathStrategy(int size, ImmutableMap<String, WeightedDc> dcs)
+    {
+        this.size = size;
+        this.dcs = dcs;
+    }
+
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ParameterizedFastPathStrategy that = (ParameterizedFastPathStrategy) o;
+        return size == that.size && Objects.equals(dcs, that.dcs);
+    }
+
+    public int hashCode()
+    {
+        return Objects.hash(size, dcs);
+    }
+
+    private static class NodeSorter implements Comparable<NodeSorter>
+    {
+        private final Node.Id id;
+        private final int sortPos;
+        private final int dcIndex;
+        private final int health;
+
+        public NodeSorter(Node.Id id, int sortPos, int dcIndex, int health)
+        {
+            this.id = id;
+            this.sortPos = sortPos;
+            this.dcIndex = dcIndex;
+            this.health = health;
+        }
+
+        @Override
+        public int compareTo(@Nonnull NodeSorter that)
+        {
+            int cmp = this.health - that.health;
+            if (cmp != 0) return cmp;
+
+            cmp = this.dcIndex - that.dcIndex;
+            if (cmp != 0) return cmp;
+
+            cmp = this.sortPos - that.sortPos;
+            if (cmp != 0) return cmp;
+
+            Invariants.checkState(this.id.equals(that.id));
+            return 0;
+        }
+    }
+
+    @Override
+    public Set<Node.Id> calculateFastPath(List<Node.Id> nodes, Set<Node.Id> unavailable, Map<Node.Id, String> dcMap)
+    {
+        List<NodeSorter> sorters = new ArrayList<>(nodes.size());
+
+        for (int i = 0, mi = nodes.size(); i < mi; i++)
+        {
+            Node.Id node = nodes.get(i);
+            String dc = dcMap.get(node);
+            int dcScore = dcs.getOrDefault(dc, WeightedDc.UNSPECIFIED).weight;
+            NodeSorter sorter = new NodeSorter(node, i, dcScore, unavailable.contains(node) ? 1 : 0);
+            sorters.add(sorter);
+        }
+
+        sorters.sort(Comparator.naturalOrder());
+
+        int slowQuorum = Shard.slowPathQuorumSize(nodes.size());
+        int fpSize = Math.max(size, slowQuorum);
+        ImmutableSet.Builder<Node.Id> builder = ImmutableSet.builder();
+
+        for (int i=0; i<fpSize; i++)
+            builder.add(sorters.get(i).id);
+
+        ImmutableSet<Node.Id> fastPath = builder.build();
+        Invariants.checkState(fastPath.size() >= slowQuorum);
+        return fastPath;
+    }
+
+    private static ConfigurationException cfe(String fmt, Object... args)
+    {
+        return new ConfigurationException(String.format(fmt, args));
+    }
+
+    static ParameterizedFastPathStrategy fromMap(Map<String, String> map)
+    {
+        if (!map.containsKey(SIZE))
+            throw cfe("fast_path must be set to 'keyspace' or 'default' or a map defining '%s' and optionally '%s'", SIZE, DCS);
+
+        int size;
+        try
+        {
+            size = Integer.parseInt(map.get(SIZE));
+        }
+        catch (NumberFormatException e)
+        {
+            throw cfe("%s must be a positive number, got %s", SIZE, map.get(SIZE));
+        }
+
+        if (size < 1)
+            throw cfe("%s must be greater than zero", SIZE);
+
+        ImmutableMap<String, WeightedDc> dcMap;
+        if (map.containsKey(DCS))
+        {
+
+            Map<String, WeightedDc> mutableDcs = new HashMap<>();
+            String dcsString = map.get(DCS);
+            if (dcsString.trim().isEmpty())
+                throw cfe("%s must specify at least one DC", DCS);
+
+            int autoIdx = 0;
+            boolean hasAuto = false;
+            boolean hasManual = false;
+            for (String dcString : COMMA_SEPARATOR.split(dcsString))
+            {
+                WeightedDc dc = WeightedDc.fromString(dcString, autoIdx++);
+                if (mutableDcs.containsKey(dc.name))
+                    throw cfe("Multiple entries for DC %s", dc.name);
+
+                if (dc.autoWeight)
+                {
+                    if (hasManual) throw cfe("Cannot mix auto and manual DC weights");
+                    hasAuto = true;
+                }
+                else
+                {
+                    if (hasAuto) throw cfe("Cannot mix auto and manual DC weights");
+                    hasManual = true;
+                }
+
+                mutableDcs.put(dc.name, dc);
+            }
+            dcMap = ImmutableMap.copyOf(mutableDcs);
+        }
+        else
+        {
+            dcMap = ImmutableMap.of();
+        }
+
+        Set<String> keys = new HashSet<>(map.keySet());
+        keys.remove(SIZE);
+        keys.remove(DCS);
+        if (!keys.isEmpty())
+            throw cfe("Unrecognized fast path options provided: ", keys);
+
+        return new ParameterizedFastPathStrategy(size, dcMap);
+    }
+
+    @Override
+    public Kind kind()
+    {
+        return Kind.PARAMETERIZED;
+    }
+
+    @VisibleForImplementation
+    public Iterable<String> dcStrings()
+    {
+        return dcs.values().stream().sorted().map(Object::toString).collect(Collectors.toList());
+    }
+
+    @Override
+    public String toString()
+    {
+        StringBuilder sb = new StringBuilder("{");
+        sb.append('\'').append(SIZE).append("':").append(size);
+        if (!dcs.isEmpty())
+            sb.append(", ").append(DCS).append(':').append('\'').append(DC_JOINER.join(dcStrings())).append('\'');
+
+        return sb.append('}').toString();
+    }
+
+    public Map<String, String> asMap()
+    {
+        Map<String, String> params = new HashMap<>();
+        params.put(Kind.KEY, kind().name());
+        params.put(SIZE, Integer.toString(size));
+        params.put(DCS, DC_JOINER.join(dcStrings()));
+        return params;
+    }
+
+    @Override
+    public String asCQL()
+    {
+        return toString();
+    }
+
+    public static final MetadataSerializer<ParameterizedFastPathStrategy> serializer = new MetadataSerializer<ParameterizedFastPathStrategy>()
+    {
+        public void serialize(ParameterizedFastPathStrategy strategy, DataOutputPlus out, Version version) throws IOException
+        {
+            out.writeUnsignedVInt32(strategy.size);
+            out.writeUnsignedVInt32(strategy.dcs.size());
+            for (WeightedDc dc : strategy.dcs.values())
+                WeightedDc.serializer.serialize(dc, out, version);
+        }
+
+        public ParameterizedFastPathStrategy deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            int size = in.readUnsignedVInt32();
+            int numDCs = in.readUnsignedVInt32();
+            ImmutableMap.Builder<String, WeightedDc> builder = ImmutableMap.builder();
+            for (int i=0; i<numDCs; i++)
+            {
+                WeightedDc dc = WeightedDc.serializer.deserialize(in, version);
+                builder.put(dc.name, dc);
+            }
+
+            return new ParameterizedFastPathStrategy(size, builder.build());
+        }
+
+        public long serializedSize(ParameterizedFastPathStrategy strategy, Version version)
+        {
+            long size = TypeSizes.sizeofUnsignedVInt(strategy.size) + TypeSizes.sizeofUnsignedVInt(strategy.dcs.size());
+            for (WeightedDc dc : strategy.dcs.values())
+                size += WeightedDc.serializer.serializedSize(dc, version);
+
+            return size;
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/fastpath/SimpleFastPathStrategy.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.fastpath;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import accord.local.Node;
+import accord.topology.Shard;
+import accord.utils.Invariants;
+
+public class SimpleFastPathStrategy implements FastPathStrategy
+{
+    static final SimpleFastPathStrategy instance = new SimpleFastPathStrategy();
+
+    private static final Map<String, String> SCHEMA_PARAMS = ImmutableMap.of(Kind.KEY, Kind.SIMPLE.name());
+
+    private SimpleFastPathStrategy() {}
+
+    @Override
+    public Set<Node.Id> calculateFastPath(List<Node.Id> nodes, Set<Node.Id> unavailable, Map<Node.Id, String> dcMap)
+    {
+        int maxFailures = Shard.maxToleratedFailures(nodes.size());
+        int discarded = 0;
+
+        ImmutableSet.Builder<Node.Id> builder = ImmutableSet.builder();
+
+        for (int i=0,mi=nodes.size(); i<mi; i++)
+        {
+            Node.Id node = nodes.get(i);
+            if (unavailable.contains(node) && discarded < maxFailures)
+            {
+                discarded++;
+                continue;
+            }
+
+            builder.add(node);
+        }
+
+        Set<Node.Id> fastPath = builder.build();
+        Invariants.checkState(fastPath.size() >= Shard.slowPathQuorumSize(nodes.size()));
+        return fastPath;
+    }
+
+    @Override
+    public Kind kind()
+    {
+        return Kind.SIMPLE;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "simple";
+    }
+
+    public Map<String, String> asMap()
+    {
+        return SCHEMA_PARAMS;
+    }
+
+    @Override
+    public String asCQL()
+    {
+        return "'simple'";
+    }
+}

src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java

@@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 
+import org.apache.cassandra.schema.TableId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -229,9 +230,9 @@ public class AccordInteropExecution implements Execute, ReadCoordinator, Maximal
     }
 
     @Override
-    public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata doNotUse, KeyspaceMetadata keyspace, Token token)
+    public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata doNotUse, KeyspaceMetadata keyspace, TableId tableId, Token token)
     {
-        AccordRoutingKey.TokenKey key = new AccordRoutingKey.TokenKey(keyspace.name, token);
+        AccordRoutingKey.TokenKey key = new AccordRoutingKey.TokenKey(tableId, token);
         Shard shard = executeTopology.forKey(key);
         Range<Token> range = ((TokenRange) shard.range).toKeyspaceRange();
 

src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java

@@ -90,8 +90,8 @@ public class CommandStoreSerializers
         public long serializedSize(R map, int version)
         {
             long size = TypeSizes.BOOL_SIZE;
-            size += TypeSizes.sizeofUnsignedVInt(size);
             int mapSize = map.size();
+            size += TypeSizes.sizeofUnsignedVInt(mapSize);
             for (int i=0; i<mapSize; i++)
             {
                 size += KeySerializers.routingKey.serializedSize(map.startAt(i), version);

src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java

@@ -32,6 +32,8 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.ArraySerializers;
 import org.apache.cassandra.utils.CollectionSerializers;
 
@@ -40,14 +42,25 @@ public class TopologySerializers
     private TopologySerializers() {}
 
     public static final NodeIdSerializer nodeId = new NodeIdSerializer();
-    public static class NodeIdSerializer implements IVersionedSerializer<Node.Id>
+    public static class NodeIdSerializer implements IVersionedSerializer<Node.Id>, MetadataSerializer<Node.Id>
     {
         private NodeIdSerializer() {}
 
+        private static void serialize(Node.Id id, DataOutputPlus out) throws IOException
+        {
+            out.writeInt(id.id);
+        }
+
         @Override
         public void serialize(Node.Id id, DataOutputPlus out, int version) throws IOException
         {
-            out.writeInt(id.id);
+            serialize(id, out);
+        }
+
+        @Override
+        public void serialize(Node.Id id, DataOutputPlus out, Version version) throws IOException
+        {
+            serialize(id, out);
         }
 
         public <V> int serialize(Node.Id id, V dst, ValueAccessor<V> accessor, int offset)
@@ -55,10 +68,21 @@ public class TopologySerializers
             return accessor.putInt(dst, offset, id.id);
         }
 
+        private static Node.Id deserialize(DataInputPlus in) throws IOException
+        {
+            return new Node.Id(in.readInt());
+        }
+
         @Override
         public Node.Id deserialize(DataInputPlus in, int version) throws IOException
         {
-            return new Node.Id(in.readInt());
+            return deserialize(in);
+        }
+
+        @Override
+        public Node.Id deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            return deserialize(in);
         }
 
         public <V> Node.Id deserialize(V src, ValueAccessor<V> accessor, int offset)
@@ -66,15 +90,21 @@ public class TopologySerializers
             return new Node.Id(accessor.getInt(src, offset));
         }
 
+        public int serializedSize()
+        {
+            return TypeSizes.INT_SIZE;  // id.id
+        }
+
         @Override
         public long serializedSize(Node.Id id, int version)
         {
             return serializedSize();
         }
 
-        public int serializedSize()
+        @Override
+        public long serializedSize(Node.Id t, Version version)
         {
-            return TypeSizes.INT_SIZE;  // id.id
+            return serializedSize();
         }
     };
 

src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java

@@ -63,7 +63,7 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
     {
         super(value);
         this.name = name;
-        this.key = new PartitionKey(value.metadata().keyspace, value.metadata().id, value.partitionKey());
+        this.key = new PartitionKey(value.metadata().id, value.partitionKey());
     }
 
     private TxnNamedRead(TxnDataName name, PartitionKey key, ByteBuffer bytes)

src/java/org/apache/cassandra/service/accord/txn/TxnQuery.java

@@ -216,6 +216,6 @@ public abstract class TxnQuery implements Query
         // and transaction statement will generate an error when it sees
         // the RetryOnNewProtocolResult
         PartitionKey partitionKey = (PartitionKey)keys.get(0);
-        return ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(epoch, partitionKey.tableId(), partitionKey.partitionKey());
+        return ConsensusRequestRouter.instance.isKeyInMigratingOrMigratedRangeFromAccord(epoch, partitionKey.table(), partitionKey.partitionKey());
     }
 }

src/java/org/apache/cassandra/service/consensus/migration/ConsensusKeyMigrationState.java

@@ -211,7 +211,7 @@ public abstract class ConsensusKeyMigrationState
 
     public static void maybeSaveAccordKeyMigrationLocally(PartitionKey partitionKey, Epoch epoch)
     {
-        TableId tableId = partitionKey.tableId();
+        TableId tableId = partitionKey.table();
         UUID tableUUID = tableId.asUUID();
         DecoratedKey dk = partitionKey.partitionKey();
         ByteBuffer key = dk.getKey();
@@ -280,7 +280,7 @@ public abstract class ConsensusKeyMigrationState
             // will soon be ready to execute, but only waits for the local replica to be ready
             // Local will only create a transaction if it can't find an existing one to wait on
             BarrierType barrierType = global ? BarrierType.global_async : BarrierType.local;
-            AccordService.instance().barrier(Seekables.of(new PartitionKey(keyspace, tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
+            AccordService.instance().barrier(Seekables.of(new PartitionKey(tableId, key)), minEpoch, requestTime, DatabaseDescriptor.getTransactionTimeout(TimeUnit.NANOSECONDS), barrierType, isForWrite);
             // We don't save the state to the cache here. Accord will notify the agent every time a barrier happens.
         }
         finally

src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java

@@ -204,6 +204,7 @@ public abstract class AbstractReadExecutor
 
         ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(metadata,
                                                                     keyspace,
+                                                                    command.metadata().id,
                                                                     command.partitionKey().getToken(),
                                                                     command.indexQueryPlan(),
                                                                     consistencyLevel,

src/java/org/apache/cassandra/service/reads/ReadCoordinator.java

@@ -29,6 +29,7 @@ import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.tcm.ClusterMetadata;
 
 public interface ReadCoordinator
@@ -40,7 +41,7 @@ public interface ReadCoordinator
             return true;
         }
 
-        public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token)
+        public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, TableId tableId, Token token)
         {
             return ReplicaLayout.forNonLocalStrategyTokenRead(metadata, keyspace, token);
         }
@@ -62,7 +63,7 @@ public interface ReadCoordinator
     };
 
     boolean localReadSupported();
-    EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token);
+    EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, TableId tableId, Token token);
     default ReadCommand maybeAllowOutOfRangeReads(ReadCommand command)
     {
         return command;

src/java/org/apache/cassandra/service/reads/range/RangeCommands.java

@@ -76,6 +76,7 @@ public class RangeCommands
         ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(),
                                                                    command.indexQueryPlan(),
                                                                    keyspace,
+                                                                   command.metadata().id(),
                                                                    consistencyLevel);
 
         if (command.isTopK())
@@ -107,7 +108,7 @@ public class RangeCommands
             Tracing.trace("Submitting range requests on {} ranges with a concurrency of {}", replicaPlans.size(), concurrencyFactor);
         }
 
-        ReplicaPlanMerger mergedReplicaPlans = new ReplicaPlanMerger(replicaPlans, keyspace, consistencyLevel);
+        ReplicaPlanMerger mergedReplicaPlans = new ReplicaPlanMerger(replicaPlans, keyspace, command.metadata().id(), consistencyLevel);
         return new RangeCommandIterator(mergedReplicaPlans,
                                         command,
                                         concurrencyFactor,
@@ -147,11 +148,12 @@ public class RangeCommands
             ReplicaPlanIterator rangeIterator = new ReplicaPlanIterator(DataRange.allData(metadata.partitioner).keyRange(),
                                                                         null,
                                                                         keyspace,
+                                                                        metadata.id,
                                                                         consistency);
 
             // Called for the side effect of running assureSufficientLiveReplicasForRead.
             // Deliberately called with an invalid vnode count in case it is used elsewhere in the future..
-            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, null, consistency, r.range(), -1));
+            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, metadata.id, null, consistency, r.range(), -1));
             return true;
         }
         catch (UnavailableException e)

src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java

@@ -37,6 +37,7 @@ import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.compatibility.TokenRingUtils;
 import org.apache.cassandra.utils.AbstractIterator;
@@ -46,6 +47,7 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
 {
     private final Keyspace keyspace;
     private final ConsistencyLevel consistency;
+    private final TableId tableId;
     private final Index.QueryPlan indexQueryPlan;
     @VisibleForTesting
     final Iterator<? extends AbstractBounds<PartitionPosition>> ranges;
@@ -54,10 +56,12 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
     ReplicaPlanIterator(AbstractBounds<PartitionPosition> keyRange,
                         @Nullable Index.QueryPlan indexQueryPlan,
                         Keyspace keyspace,
+                        TableId tableId,
                         ConsistencyLevel consistency)
     {
         this.indexQueryPlan = indexQueryPlan;
         this.keyspace = keyspace;
+        this.tableId = tableId;
         this.consistency = consistency;
 
         ReplicationParams replication = keyspace.getMetadata().params.replication;
@@ -82,7 +86,7 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
         if (!ranges.hasNext())
             return endOfData();
 
-        return ReplicaPlans.forRangeRead(keyspace, indexQueryPlan, consistency, ranges.next(), 1);
+        return ReplicaPlans.forRangeRead(keyspace, tableId, indexQueryPlan, consistency, ranges.next(), 1);
     }
 
     /**

src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java

@@ -27,6 +27,7 @@ import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.utils.AbstractIterator;
 
@@ -34,11 +35,13 @@ class ReplicaPlanMerger extends AbstractIterator<ReplicaPlan.ForRangeRead>
 {
     private final Keyspace keyspace;
     private final ConsistencyLevel consistency;
+    private final TableId tableId;
     private final PeekingIterator<ReplicaPlan.ForRangeRead> ranges;
 
-    ReplicaPlanMerger(Iterator<ReplicaPlan.ForRangeRead> iterator, Keyspace keyspace, ConsistencyLevel consistency)
+    ReplicaPlanMerger(Iterator<ReplicaPlan.ForRangeRead> iterator, Keyspace keyspace, TableId tableId, ConsistencyLevel consistency)
     {
         this.keyspace = keyspace;
+        this.tableId = tableId;
         this.consistency = consistency;
         this.ranges = Iterators.peekingIterator(iterator);
     }
@@ -66,7 +69,7 @@ class ReplicaPlanMerger extends AbstractIterator<ReplicaPlan.ForRangeRead>
                 break;
 
             ReplicaPlan.ForRangeRead next = ranges.peek();
-            ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(metadata, keyspace, consistency, current, next);
+            ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(metadata, keyspace, tableId, consistency, current, next);
             if (merged == null)
                 break;
 

src/java/org/apache/cassandra/streaming/StreamPlan.java

@@ -24,6 +24,8 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.utils.TimeUUID;
 
 import static com.google.common.collect.Iterables.all;
@@ -225,4 +227,31 @@ public class StreamPlan
     {
         return coordinator;
     }
+
+    /**
+     * Returns an array containing the non-accord tables for the given keyspace. Since the relevant StreamPlan methods
+     * interpret an empty array to mean all tables, null is returned if there are no non-accord tables in
+     * the given keyspace
+     * @param ksm
+     * @return
+     */
+    public static String[] nonAccordTablesForKeyspace(KeyspaceMetadata ksm)
+    {
+        String[] result = ksm.tables.stream()
+                                    .filter(tbl -> !AccordService.instance().isAccordManagedTable(tbl.id))
+                                    .map(tbl -> tbl.name)
+                                    .toArray(String[]::new);
+
+        return result.length > 0 ? result : null;
+    }
+
+    public static boolean hasNonAccordTables(KeyspaceMetadata ksm)
+    {
+        return ksm.tables.stream().anyMatch(tbl -> !AccordService.instance().isAccordManagedTable(tbl.id));
+    }
+
+    public static boolean hasAccordTables(KeyspaceMetadata ksm)
+    {
+        return ksm.tables.stream().anyMatch(tbl -> AccordService.instance().isAccordManagedTable(tbl.id));
+    }
 }

src/java/org/apache/cassandra/tcm/ClusterMetadata.java

@@ -32,10 +32,12 @@ import java.util.Set;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import accord.local.Node;
 import org.apache.cassandra.config.CassandraRelevantProperties;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.dht.IPartitioner;
@@ -59,13 +61,9 @@ import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationS
 import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState.TableMigrationState;
 import org.apache.cassandra.tcm.extensions.ExtensionKey;
 import org.apache.cassandra.tcm.extensions.ExtensionValue;
-import org.apache.cassandra.tcm.membership.Directory;
-import org.apache.cassandra.tcm.membership.Location;
-import org.apache.cassandra.tcm.membership.NodeAddresses;
-import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.membership.NodeState;
-import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.AccordKeyspaces;
+import org.apache.cassandra.service.accord.AccordFastPath;
+import org.apache.cassandra.tcm.membership.*;
+import org.apache.cassandra.tcm.ownership.AccordTables;
 import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator;
@@ -82,6 +80,7 @@ import org.apache.cassandra.utils.Pair;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
 import static org.apache.cassandra.db.TypeSizes.sizeof;
+import static org.apache.cassandra.tcm.serialization.Version.V2;
 
 public class ClusterMetadata
 {
@@ -97,7 +96,8 @@ public class ClusterMetadata
     public final Directory directory;
     public final TokenMap tokenMap;
     public final DataPlacements placements;
-    public final AccordKeyspaces accordKeyspaces;
+    public final AccordTables accordTables;
+    public final AccordFastPath accordFastPath;
     public final LockedRanges lockedRanges;
     public final InProgressSequences inProgressSequences;
     public final ConsensusMigrationState consensusMigrationState;
@@ -133,7 +133,8 @@ public class ClusterMetadata
              directory,
              new TokenMap(partitioner),
              DataPlacements.EMPTY,
-             AccordKeyspaces.EMPTY,
+             AccordTables.EMPTY,
+             AccordFastPath.EMPTY,
              LockedRanges.EMPTY,
              InProgressSequences.EMPTY,
              ConsensusMigrationState.EMPTY,
@@ -146,7 +147,8 @@ public class ClusterMetadata
                            Directory directory,
                            TokenMap tokenMap,
                            DataPlacements placements,
-                           AccordKeyspaces accordKeyspaces,
+                           AccordTables accordTables,
+                           AccordFastPath accordFastPath,
                            LockedRanges lockedRanges,
                            InProgressSequences inProgressSequences,
                            ConsensusMigrationState consensusMigrationState,
@@ -159,7 +161,8 @@ public class ClusterMetadata
              directory,
              tokenMap,
              placements,
-             accordKeyspaces,
+             accordTables,
+             accordFastPath,
              lockedRanges,
              inProgressSequences,
              consensusMigrationState,
@@ -173,7 +176,8 @@ public class ClusterMetadata
                            Directory directory,
                            TokenMap tokenMap,
                            DataPlacements placements,
-                           AccordKeyspaces accordKeyspaces,
+                           AccordTables accordTables,
+                           AccordFastPath accordFastPath,
                            LockedRanges lockedRanges,
                            InProgressSequences inProgressSequences,
                            ConsensusMigrationState consensusMigrationState,
@@ -190,7 +194,8 @@ public class ClusterMetadata
         this.directory = directory;
         this.tokenMap = tokenMap;
         this.placements = placements;
-        this.accordKeyspaces = accordKeyspaces;
+        this.accordTables = accordTables;
+        this.accordFastPath = accordFastPath;
         this.lockedRanges = lockedRanges;
         this.inProgressSequences = inProgressSequences;
         this.consensusMigrationState = consensusMigrationState;
@@ -200,12 +205,12 @@ public class ClusterMetadata
 
     public ClusterMetadata withDirectory(Directory directory)
     {
-        return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordKeyspaces, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
+        return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordTables, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
     }
 
     public ClusterMetadata withPlacements(DataPlacements placements)
     {
-        return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordKeyspaces, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
+        return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordTables, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
     }
 
     public Set<InetAddressAndPort> fullCMSMembers()
@@ -257,7 +262,8 @@ public class ClusterMetadata
                                    capLastModified(directory, epoch),
                                    capLastModified(tokenMap, epoch),
                                    capLastModified(placements, epoch),
-                                   capLastModified(accordKeyspaces, epoch),
+                                   capLastModified(accordTables, epoch),
+                                   capLastModified(accordFastPath, epoch),
                                    capLastModified(lockedRanges, epoch),
                                    capLastModified(inProgressSequences, epoch),
                                    capLastModified(consensusMigrationState, epoch),
@@ -279,7 +285,8 @@ public class ClusterMetadata
                                    directory,
                                    tokenMap,
                                    placements,
-                                   accordKeyspaces,
+                                   accordTables,
+                                   accordFastPath,
                                    lockedRanges,
                                    inProgressSequences,
                                    consensusMigrationState,
@@ -406,7 +413,8 @@ public class ClusterMetadata
         private Directory directory;
         private TokenMap tokenMap;
         private DataPlacements placements;
-        private AccordKeyspaces accordKeyspaces;
+        private AccordTables accordTables;
+        private AccordFastPath accordFastPath;
         private LockedRanges lockedRanges;
         private InProgressSequences inProgressSequences;
         private ConsensusMigrationState consensusMigrationState;
@@ -422,7 +430,8 @@ public class ClusterMetadata
             this.directory = metadata.directory;
             this.tokenMap = metadata.tokenMap;
             this.placements = metadata.placements;
-            this.accordKeyspaces = metadata.accordKeyspaces;
+            this.accordTables = metadata.accordTables;
+            this.accordFastPath = metadata.accordFastPath;
             this.lockedRanges = metadata.lockedRanges;
             this.inProgressSequences = metadata.inProgressSequences;
             this.consensusMigrationState = metadata.consensusMigrationState;
@@ -543,9 +552,15 @@ public class ClusterMetadata
             return this;
         }
 
-        public Transformer withAccordKeyspace(String keyspace)
+        public Transformer withAccordTable(TableId table)
         {
-            accordKeyspaces = accordKeyspaces.with(keyspace);
+            accordTables = accordTables.with(table);
+            return this;
+        }
+
+        public Transformer withFastPathStatusSince(Node.Id node, AccordFastPath.Status status, long updateTimeMillis, long updateDelayMillis)
+        {
+            accordFastPath = accordFastPath.withNodeStatusSince(node, status, updateTimeMillis, updateDelayMillis);
             return this;
         }
 
@@ -640,6 +655,9 @@ public class ClusterMetadata
             {
                 modifiedKeys.add(MetadataKeys.NODE_DIRECTORY);
                 directory = directory.withLastModified(epoch);
+
+                for (NodeId peer : Sets.difference(base.directory.peerIds(), directory.peerIds()))
+                    accordFastPath = accordFastPath.withoutNode(peer);
             }
 
             if (tokenMap != base.tokenMap)
@@ -660,10 +678,16 @@ public class ClusterMetadata
                 placements = placements.withLastModified(epoch);
             }
 
-            if (accordKeyspaces != base.accordKeyspaces)
+            if (accordTables != base.accordTables)
             {
-                modifiedKeys.add(MetadataKeys.ACCORD_KEYSPACES);
-                accordKeyspaces = accordKeyspaces.withLastModified(epoch);
+                modifiedKeys.add(MetadataKeys.ACCORD_TABLES);
+                accordTables = accordTables.withLastModified(epoch);
+            }
+
+            if (accordFastPath != base.accordFastPath)
+            {
+                modifiedKeys.add(MetadataKeys.ACCORD_FAST_PATH);
+                accordFastPath = accordFastPath.withLastModified(epoch);
             }
 
             if (lockedRanges != base.lockedRanges)
@@ -691,7 +715,8 @@ public class ClusterMetadata
                                                        directory,
                                                        tokenMap,
                                                        placements,
-                                                       accordKeyspaces,
+                                                       accordTables,
+                                                       accordFastPath,
                                                        lockedRanges,
                                                        inProgressSequences,
                                                        consensusMigrationState,
@@ -708,7 +733,8 @@ public class ClusterMetadata
                                        directory,
                                        tokenMap,
                                        placements,
-                                       accordKeyspaces,
+                                       accordTables,
+                                       accordFastPath,
                                        lockedRanges,
                                        inProgressSequences,
                                        consensusMigrationState,
@@ -726,6 +752,7 @@ public class ClusterMetadata
                    ", directory=" + schema +
                    ", tokenMap=" + tokenMap +
                    ", placement=" + placements +
+                   ", availability=" + accordFastPath +
                    ", lockedRanges=" + lockedRanges +
                    ", inProgressSequences=" + inProgressSequences +
                    ", consensusMigrationState=" + consensusMigrationState +
@@ -839,7 +866,7 @@ public class ClusterMetadata
                directory.equals(that.directory) &&
                tokenMap.equals(that.tokenMap) &&
                placements.equals(that.placements) &&
-               accordKeyspaces.equals(that.accordKeyspaces) &&
+               accordTables.equals(that.accordTables) &&
                lockedRanges.equals(that.lockedRanges) &&
                inProgressSequences.equals(that.inProgressSequences) &&
                consensusMigrationState.equals(that.consensusMigrationState) &&
@@ -891,7 +918,7 @@ public class ClusterMetadata
     @Override
     public int hashCode()
     {
-        return Objects.hash(epoch, schema, directory, tokenMap, placements, accordKeyspaces, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
+        return Objects.hash(epoch, schema, directory, tokenMap, placements, accordTables, lockedRanges, inProgressSequences, consensusMigrationState, extensions);
     }
 
     public static ClusterMetadata current()
@@ -968,7 +995,11 @@ public class ClusterMetadata
             Directory.serializer.serialize(metadata.directory, out, version);
             TokenMap.serializer.serialize(metadata.tokenMap, out, version);
             DataPlacements.serializer.serialize(metadata.placements, out, version);
-            AccordKeyspaces.serializer.serialize(metadata.accordKeyspaces, out, version);
+            if (version.isAtLeast(V2))
+            {
+                AccordTables.serializer.serialize(metadata.accordTables, out, version);
+                AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version);
+            }
             LockedRanges.serializer.serialize(metadata.lockedRanges, out, version);
             InProgressSequences.serializer.serialize(metadata.inProgressSequences, out, version);
             ConsensusMigrationState.serializer.serialize(metadata.consensusMigrationState, out, version);
@@ -1006,7 +1037,18 @@ public class ClusterMetadata
             Directory dir = Directory.serializer.deserialize(in, version);
             TokenMap tokenMap = TokenMap.serializer.deserialize(in, version);
             DataPlacements placements = DataPlacements.serializer.deserialize(in, version);
-            AccordKeyspaces accordKeyspaces = AccordKeyspaces.serializer.deserialize(in, version);
+            AccordTables accordTables;
+            AccordFastPath accordFastPath;
+            if (version.isAtLeast(V2))
+            {
+                accordTables = AccordTables.serializer.deserialize(in, version);
+                accordFastPath = AccordFastPath.serializer.deserialize(in, version);
+            }
+            else
+            {
+                accordTables = AccordTables.EMPTY;
+                accordFastPath = AccordFastPath.EMPTY;
+            }
             LockedRanges lockedRanges = LockedRanges.serializer.deserialize(in, version);
             InProgressSequences ips = InProgressSequences.serializer.deserialize(in, version);
             ConsensusMigrationState consensusMigrationState = ConsensusMigrationState.serializer.deserialize(in, version);
@@ -1026,7 +1068,8 @@ public class ClusterMetadata
                                        dir,
                                        tokenMap,
                                        placements,
-                                       accordKeyspaces,
+                                       accordTables,
+                                       accordFastPath,
                                        lockedRanges,
                                        ips,
                                        consensusMigrationState,
@@ -1050,10 +1093,17 @@ public class ClusterMetadata
                     Directory.serializer.serializedSize(metadata.directory, version) +
                     TokenMap.serializer.serializedSize(metadata.tokenMap, version) +
                     DataPlacements.serializer.serializedSize(metadata.placements, version) +
-                    AccordKeyspaces.serializer.serializedSize(metadata.accordKeyspaces, version) +
-                    LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) +
-                    InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version) +
                     ConsensusMigrationState.serializer.serializedSize(metadata.consensusMigrationState, version);
+                    DataPlacements.serializer.serializedSize(metadata.placements, version);
+
+            if (version.isAtLeast(V2))
+            {
+                size += AccordTables.serializer.serializedSize(metadata.accordTables, version) +
+                        AccordFastPath.serializer.serializedSize(metadata.accordFastPath, version);
+            }
+
+            size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) +
+                    InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version);
 
             return size;
         }

src/java/org/apache/cassandra/tcm/MetadataKeys.java

@@ -39,7 +39,8 @@ public class MetadataKeys
     public static final MetadataKey NODE_DIRECTORY          = make(CORE_NS, "membership", "node_directory");
     public static final MetadataKey TOKEN_MAP               = make(CORE_NS, "ownership", "token_map");
     public static final MetadataKey DATA_PLACEMENTS         = make(CORE_NS, "ownership", "data_placements");
-    public static final MetadataKey ACCORD_KEYSPACES        = make(CORE_NS, "ownership", "accord_keyspaces");
+    public static final MetadataKey ACCORD_TABLES           = make(CORE_NS, "ownership", "accord_tables");
+    public static final MetadataKey ACCORD_FAST_PATH        = make(CORE_NS, "ownership", "accord_fast_path");
     public static final MetadataKey LOCKED_RANGES           = make(CORE_NS, "sequences", "locked_ranges");
     public static final MetadataKey IN_PROGRESS_SEQUENCES   = make(CORE_NS, "sequences", "in_progress");
     public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state");
@@ -48,7 +49,8 @@ public class MetadataKeys
                                                                                   NODE_DIRECTORY,
                                                                                   TOKEN_MAP,
                                                                                   DATA_PLACEMENTS,
-                                                                                  ACCORD_KEYSPACES,
+                                                                                  ACCORD_TABLES,
+                                                                                  ACCORD_FAST_PATH,
                                                                                   LOCKED_RANGES,
                                                                                   IN_PROGRESS_SEQUENCES,
                                                                                   CONSENSUS_MIGRATION_STATE);

src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java

@@ -28,12 +28,13 @@ import org.apache.cassandra.schema.DistributedMetadataLogKeyspace;
 import org.apache.cassandra.schema.DistributedSchema;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.service.accord.AccordFastPath;
 import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState;
 import org.apache.cassandra.tcm.Commit.Replicator;
 import org.apache.cassandra.tcm.log.Entry;
 import org.apache.cassandra.tcm.log.LocalLog;
 import org.apache.cassandra.tcm.membership.Directory;
-import org.apache.cassandra.tcm.ownership.AccordKeyspaces;
+import org.apache.cassandra.tcm.ownership.AccordTables;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.PlacementProvider;
 import org.apache.cassandra.tcm.ownership.TokenMap;
@@ -174,7 +175,8 @@ public class StubClusterMetadataService extends ClusterMetadataService
                                               Directory.EMPTY,
                                               new TokenMap(partitioner),
                                               DataPlacements.EMPTY,
-                                              AccordKeyspaces.EMPTY,
+                                              AccordTables.EMPTY,
+                                              AccordFastPath.EMPTY,
                                               LockedRanges.EMPTY,
                                               InProgressSequences.EMPTY,
                                               ConsensusTableMigrationState.ConsensusMigrationState.EMPTY,

src/java/org/apache/cassandra/tcm/Transformation.java

@@ -38,7 +38,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
-import org.apache.cassandra.tcm.transformations.AddAccordKeyspace;
+import org.apache.cassandra.tcm.transformations.AddAccordTable;
 import org.apache.cassandra.tcm.transformations.AlterSchema;
 import org.apache.cassandra.tcm.transformations.AlterTopology;
 import org.apache.cassandra.tcm.transformations.Assassinate;
@@ -51,6 +51,7 @@ import org.apache.cassandra.tcm.transformations.PrepareJoin;
 import org.apache.cassandra.tcm.transformations.PrepareLeave;
 import org.apache.cassandra.tcm.transformations.PrepareMove;
 import org.apache.cassandra.tcm.transformations.PrepareReplace;
+import org.apache.cassandra.tcm.transformations.ReconfigureAccordFastPath;
 import org.apache.cassandra.tcm.transformations.Register;
 import org.apache.cassandra.tcm.transformations.SetConsensusMigrationTargetProtocol;
 import org.apache.cassandra.tcm.transformations.Startup;
@@ -239,10 +240,11 @@ public interface Transformation
         CANCEL_CMS_RECONFIGURATION(34, () -> CancelCMSReconfiguration.serializer),
         ALTER_TOPOLOGY(35, () -> AlterTopology.serializer),
 
-        ADD_ACCORD_KEYSPACE(36, () -> AddAccordKeyspace.serializer),
-        BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(37, () -> BeginConsensusMigrationForTableAndRange.serializer),
-        MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(38, () -> MaybeFinishConsensusMigrationForTableAndRange.serializer),
-        SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL(39, () -> SetConsensusMigrationTargetProtocol.serializer)
+        ADD_ACCORD_TABLE(36, () -> AddAccordTable.serializer),
+        UPDATE_AVAILABILITY(37, () -> ReconfigureAccordFastPath.serializer),
+        BEGIN_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(38, () -> BeginConsensusMigrationForTableAndRange.serializer),
+        MAYBE_FINISH_CONSENSUS_MIGRATION_FOR_TABLE_AND_RANGE(39, () -> MaybeFinishConsensusMigrationForTableAndRange.serializer),
+        SET_CONSENSUS_MIGRATION_TARGET_PROTOCOL(40, () -> SetConsensusMigrationTargetProtocol.serializer)
         ;
 
         private final Supplier<AsymmetricMetadataSerializer<Transformation, ? extends Transformation>> serializer;

src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java

@@ -60,13 +60,14 @@ import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MultiStepOperation;
 import org.apache.cassandra.tcm.extensions.ExtensionKey;
 import org.apache.cassandra.tcm.extensions.ExtensionValue;
+import org.apache.cassandra.service.accord.AccordFastPath;
 import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.Location;
 import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.AccordKeyspaces;
+import org.apache.cassandra.tcm.ownership.AccordTables;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.TokenMap;
 import org.apache.cassandra.tcm.ownership.UniformRangePlacement;
@@ -296,7 +297,8 @@ public class GossipHelper
                                    Directory.EMPTY,
                                    new TokenMap(DatabaseDescriptor.getPartitioner()),
                                    DataPlacements.empty(),
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    LockedRanges.EMPTY,
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,
@@ -385,7 +387,8 @@ public class GossipHelper
                                                                       directory,
                                                                       tokenMap,
                                                                       DataPlacements.empty(),
-                                                                      AccordKeyspaces.EMPTY,
+                                                                      AccordTables.EMPTY,
+                                                                      AccordFastPath.EMPTY,
                                                                       LockedRanges.EMPTY,
                                                                       InProgressSequences.EMPTY,
                                                                       ConsensusMigrationState.EMPTY,
@@ -399,7 +402,8 @@ public class GossipHelper
                                    directory,
                                    tokenMap,
                                    placements,
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    LockedRanges.EMPTY,
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,

src/java/org/apache/cassandra/tcm/ownership/AccordKeyspaces.java

@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.tcm.ownership;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.tcm.Epoch;
-import org.apache.cassandra.tcm.MetadataValue;
-import org.apache.cassandra.tcm.serialization.MetadataSerializer;
-import org.apache.cassandra.tcm.serialization.Version;
-
-public class AccordKeyspaces implements MetadataValue<AccordKeyspaces>
-{
-    public static final AccordKeyspaces EMPTY = new AccordKeyspaces(Epoch.EMPTY, ImmutableSet.of());
-    private final Epoch lastModified;
-    private final ImmutableSet<String> keyspaces;
-
-    public AccordKeyspaces(Epoch lastModified, ImmutableSet<String> keyspaces)
-    {
-        this.lastModified = lastModified;
-        this.keyspaces = keyspaces;
-    }
-
-    public String toString()
-    {
-        return "AccordKeyspaces{" + lastModified + keyspaces + '}';
-    }
-
-    public AccordKeyspaces withLastModified(Epoch epoch)
-    {
-        return new AccordKeyspaces(epoch, keyspaces);
-    }
-
-    public Epoch lastModified()
-    {
-        return lastModified;
-    }
-
-    public boolean contains(String keyspace)
-    {
-        return keyspaces.contains(keyspace);
-    }
-
-    public AccordKeyspaces with(String keyspace)
-    {
-        if (keyspaces.contains(keyspace))
-            return this;
-
-        return new AccordKeyspaces(lastModified, ImmutableSet.<String>builder().addAll(keyspaces).add(keyspace).build());
-    }
-
-    public static final MetadataSerializer<AccordKeyspaces> serializer = new MetadataSerializer<AccordKeyspaces>()
-    {
-        public void serialize(AccordKeyspaces accordKeyspaces, DataOutputPlus out, Version version) throws IOException
-        {
-            int size = accordKeyspaces.keyspaces.size();
-            out.writeInt(size);
-            String[] keyspaces = new String[size];
-            accordKeyspaces.keyspaces.toArray(keyspaces);
-            Arrays.sort(keyspaces);
-            for (String keyspace : keyspaces)
-                out.writeUTF(keyspace);
-            Epoch.serializer.serialize(accordKeyspaces.lastModified, out, version);
-        }
-
-        public AccordKeyspaces deserialize(DataInputPlus in, Version version) throws IOException
-        {
-            int size = in.readInt();
-            ImmutableSet.Builder<String> builder = ImmutableSet.builder();
-            for (int i=0; i<size; i++)
-                builder.add(in.readUTF());
-            Epoch lastModificed = Epoch.serializer.deserialize(in, version);
-
-            return new AccordKeyspaces(lastModificed, builder.build());
-        }
-
-        public long serializedSize(AccordKeyspaces accordKeyspaces, Version version)
-        {
-            long size = TypeSizes.INT_SIZE;
-            for (String keyspace : accordKeyspaces.keyspaces)
-                size += TypeSizes.sizeof(keyspace);
-            size += Epoch.serializer.serializedSize(accordKeyspaces.lastModified, version);
-            return size;
-        }
-    };
-}

src/java/org/apache/cassandra/tcm/ownership/AccordTables.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.ownership;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.MetadataValue;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+public class AccordTables implements MetadataValue<AccordTables>
+{
+    public static final AccordTables EMPTY = new AccordTables(Epoch.EMPTY, ImmutableSet.of());
+    private final Epoch lastModified;
+    private final ImmutableSet<TableId> tables;
+
+    public AccordTables(Epoch lastModified, ImmutableSet<TableId> tables)
+    {
+        this.lastModified = lastModified;
+        this.tables = tables;
+    }
+
+    public String toString()
+    {
+        return "AccordTables{" + lastModified + ", " + tables + '}';
+    }
+
+    public AccordTables withLastModified(Epoch epoch)
+    {
+        return new AccordTables(epoch, tables);
+    }
+
+    public Epoch lastModified()
+    {
+        return lastModified;
+    }
+
+    public boolean contains(TableId table)
+    {
+        return tables.contains(table);
+    }
+
+    public AccordTables with(TableId table)
+    {
+        if (tables.contains(table))
+            return this;
+
+        return new AccordTables(lastModified, ImmutableSet.<TableId>builder().addAll(tables).add(table).build());
+    }
+
+    public static final MetadataSerializer<AccordTables> serializer = new MetadataSerializer<AccordTables>()
+    {
+        public void serialize(AccordTables accordTables, DataOutputPlus out, Version version) throws IOException
+        {
+            int size = accordTables.tables.size();
+            out.writeUnsignedVInt32(size);
+            TableId[] tables = new TableId[size];
+            accordTables.tables.toArray(tables);
+            Arrays.sort(tables);
+            for (TableId table : tables)
+                table.serialize(out);
+            Epoch.serializer.serialize(accordTables.lastModified, out, version);
+        }
+
+        public AccordTables deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            int size = in.readUnsignedVInt32();
+            ImmutableSet.Builder<TableId> builder = ImmutableSet.builder();
+            for (int i=0; i<size; i++)
+                builder.add(TableId.deserialize(in));
+            Epoch lastModified = Epoch.serializer.deserialize(in, version);
+
+            return new AccordTables(lastModified, builder.build());
+        }
+
+        public long serializedSize(AccordTables accordTables, Version version)
+        {
+            long size = TypeSizes.sizeofUnsignedVInt(accordTables.tables.size());
+            for (TableId table : accordTables.tables)
+                size += table.serializedSize();
+            size += Epoch.serializer.serializedSize(accordTables.lastModified, version);
+            return size;
+        }
+    };
+}

src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java

@@ -33,6 +33,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.Directory;
@@ -94,7 +95,7 @@ public class CancelCMSReconfiguration implements Transformation
 
             // Also update schema with the corrected params
             KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
-            KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement));
+            KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple()));
             transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)));
         }
 

src/java/org/apache/cassandra/tcm/sequences/Move.java

@@ -229,12 +229,13 @@ public class Move extends MultiStepOperation<Epoch>
 
                     for (KeyspaceMetadata ks : keyspaces)
                     {
-                        if (AccordService.instance().isAccordManagedKeyspace(ks.name))
-                            continue;
                         ReplicationParams replicationParams = ks.params.replication;
-                        if (replicationParams.isMeta())
+                        if (replicationParams.isMeta() || !StreamPlan.hasNonAccordTables(ks))
                             continue;
+
                         EndpointsByReplica endpoints = movementMap.get(replicationParams);
+
+                        String[] cfNames = StreamPlan.nonAccordTablesForKeyspace(ks);
                         for (Map.Entry<Replica, Replica> e : endpoints.flattenEntries())
                         {
                             Replica destination = e.getKey();
@@ -242,13 +243,13 @@ public class Move extends MultiStepOperation<Epoch>
                             logger.info("Stream source: {} destination: {}", source, destination);
                             assert !source.endpoint().equals(destination.endpoint()) : String.format("Source %s should not be the same as destionation %s", source, destination);
                             if (source.isSelf())
-                                streamPlan.transferRanges(destination.endpoint(), ks.name, RangesAtEndpoint.of(destination));
+                                streamPlan.transferRanges(destination.endpoint(), ks.name, RangesAtEndpoint.of(destination), cfNames);
                             else if (destination.isSelf())
                             {
                                 if (destination.isFull())
-                                    streamPlan.requestRanges(source.endpoint(), ks.name, RangesAtEndpoint.of(destination), RangesAtEndpoint.empty(destination.endpoint()));
+                                    streamPlan.requestRanges(source.endpoint(), ks.name, RangesAtEndpoint.of(destination), RangesAtEndpoint.empty(destination.endpoint()), cfNames);
                                 else
-                                    streamPlan.requestRanges(source.endpoint(), ks.name, RangesAtEndpoint.empty(destination.endpoint()), RangesAtEndpoint.of(destination));
+                                    streamPlan.requestRanges(source.endpoint(), ks.name, RangesAtEndpoint.empty(destination.endpoint()), RangesAtEndpoint.of(destination), cfNames);
                             }
                             else
                                 throw new IllegalStateException("Node should be either source or destination in the movement map " + endpoints);

src/java/org/apache/cassandra/tcm/serialization/Version.java

@@ -36,6 +36,7 @@ public enum Version
     /**
      *  - Added version to PlacementForRange serializer
      *  - Serialize MemtableParams when serializing TableParams
+     *  - Added AccordFastPath
      */
     V2(2),
     /**

src/java/org/apache/cassandra/tcm/transformations/AddAccordTable.java

@@ -20,59 +20,72 @@ package org.apache.cassandra.tcm.transformations;
 
 import java.io.IOException;
 
-import org.apache.cassandra.db.TypeSizes;
+import accord.utils.Invariants;
 import org.apache.cassandra.exceptions.ExceptionCode;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
 // TODO (expected, interop): improve mechanism for adding tables (probably want table level granularity, option to not-auto-add, and option to remove)
-public class AddAccordKeyspace implements Transformation
+public class AddAccordTable implements Transformation
 {
-    private final String keyspace;
+    private final TableId table;
 
-    public AddAccordKeyspace(String keyspace)
+    public AddAccordTable(TableId table)
     {
-        this.keyspace = keyspace;
+        this.table = table;
     }
 
     public Kind kind()
     {
-        return Kind.ADD_ACCORD_KEYSPACE;
+        return Kind.ADD_ACCORD_TABLE;
     }
 
     @Override
     public Result execute(ClusterMetadata metadata)
     {
-        if (metadata.accordKeyspaces.contains(keyspace))
-            return new Rejected(ExceptionCode.ALREADY_EXISTS, keyspace + " is already an accord keyspaces");
+        if (metadata.accordTables.contains(table))
+            return new Rejected(ExceptionCode.ALREADY_EXISTS, table + " is already an accord table");
 
-        return Transformation.success(metadata.transformer().withAccordKeyspace(keyspace), LockedRanges.AffectedRanges.EMPTY);
+        return Transformation.success(metadata.transformer().withAccordTable(table), LockedRanges.AffectedRanges.EMPTY);
     }
 
-    public static final AsymmetricMetadataSerializer<Transformation, AddAccordKeyspace> serializer = new AsymmetricMetadataSerializer<Transformation, AddAccordKeyspace>()
+    public static void addTable(TableId table)
+    {
+        ClusterMetadataService.instance().commit(new AddAccordTable(table),
+                                                 metadata -> null,
+                                                 (code, message) -> {
+                                                     Invariants.checkState(code == ExceptionCode.ALREADY_EXISTS,
+                                                                           "Expected %s, got %s", ExceptionCode.ALREADY_EXISTS, code);
+                                                     return null;
+                                                 });
+    }
+
+    public static final AsymmetricMetadataSerializer<Transformation, AddAccordTable> serializer = new AsymmetricMetadataSerializer<Transformation, AddAccordTable>()
     {
         public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
         {
-            assert t instanceof AddAccordKeyspace;
-            AddAccordKeyspace addKeyspace = (AddAccordKeyspace) t;
-            out.writeUTF(addKeyspace.keyspace);
+            assert t instanceof AddAccordTable;
+            AddAccordTable addTable = (AddAccordTable) t;
+            addTable.table.serialize(out);
         }
 
-        public AddAccordKeyspace deserialize(DataInputPlus in, Version version) throws IOException
+        public AddAccordTable deserialize(DataInputPlus in, Version version) throws IOException
         {
-            return new AddAccordKeyspace(in.readUTF());
+            return new AddAccordTable(TableId.deserialize(in));
         }
 
         public long serializedSize(Transformation t, Version version)
         {
-            assert t instanceof AddAccordKeyspace;
-            AddAccordKeyspace addKeyspace = (AddAccordKeyspace) t;
-            return TypeSizes.sizeof(addKeyspace.keyspace);
+            assert t instanceof AddAccordTable;
+            AddAccordTable addTable = (AddAccordTable) t;
+            return addTable.table.serializedSize();
         }
     };
 }

src/java/org/apache/cassandra/tcm/transformations/ReconfigureAccordFastPath.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm.transformations;
+
+import java.io.IOException;
+
+import accord.local.Node;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.ExceptionCode;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.AccordFastPath;
+import org.apache.cassandra.service.accord.serializers.TopologySerializers;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Transformation;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+
+public class ReconfigureAccordFastPath implements Transformation
+{
+    private final Node.Id node;
+    private final AccordFastPath.Status status;
+    private final long updateTimeMillis;
+    private final long updateDelayMillis;
+
+    public ReconfigureAccordFastPath(Node.Id node, AccordFastPath.Status status, long updateTimeMillis, long updateDelayMillis)
+    {
+        this.node = node;
+        this.status = status;
+        this.updateTimeMillis = updateTimeMillis;
+        this.updateDelayMillis = updateDelayMillis;
+    }
+
+    public Kind kind()
+    {
+        return Kind.UPDATE_AVAILABILITY;
+    }
+
+    public Result execute(ClusterMetadata metadata)
+    {
+        try
+        {
+            return Transformation.success(metadata.transformer().withFastPathStatusSince(node, status, updateTimeMillis, updateDelayMillis), LockedRanges.AffectedRanges.EMPTY);
+        }
+        catch (InvalidRequestException e)
+        {
+            return new Rejected(ExceptionCode.INVALID, e.getMessage());
+        }
+    }
+
+    public static final AsymmetricMetadataSerializer<Transformation, ReconfigureAccordFastPath> serializer = new AsymmetricMetadataSerializer<Transformation, ReconfigureAccordFastPath>()
+    {
+        public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
+        {
+            ReconfigureAccordFastPath update = (ReconfigureAccordFastPath) t;
+            TopologySerializers.nodeId.serialize(update.node, out, version);
+            AccordFastPath.Status.serializer.serialize(update.status, out, version);
+            out.writeUnsignedVInt(update.updateTimeMillis);
+            out.writeUnsignedVInt(update.updateDelayMillis);
+
+        }
+
+        public ReconfigureAccordFastPath deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            return new ReconfigureAccordFastPath(TopologySerializers.nodeId.deserialize(in, version),
+                                          AccordFastPath.Status.serializer.deserialize(in, version),
+                                          in.readUnsignedVInt(), in.readUnsignedVInt());
+        }
+
+        public long serializedSize(Transformation t, Version version)
+        {
+            ReconfigureAccordFastPath update = (ReconfigureAccordFastPath) t;
+            return TopologySerializers.nodeId.serializedSize(update.node, version) +
+                   AccordFastPath.Status.serializer.serializedSize(update.status, version) +
+                   TypeSizes.sizeofUnsignedVInt(update.updateTimeMillis) +
+                   TypeSizes.sizeofUnsignedVInt(update.updateDelayMillis);
+        }
+    };
+}

src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java

@@ -45,6 +45,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.NodeId;
@@ -263,7 +264,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
             // In a complex reconfiguration, in addition to initiating the sequence of membership changes,
             // we're modifying the replication params of the metadata keyspace so we supply a function to do that
             KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
-            KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams));
+            KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple()));
 
             return executeInternal(prev,
                                    transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams))

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -1332,12 +1332,16 @@ public class NodeProbe implements AutoCloseable
         return ssProxy.getNonLocalStrategyKeyspaces();
     }
 
-
-    public List<String> getAccordManagedKeyspace()
+    public List<String> getAccordManagedKeyspaces()
     {
         return ssProxy.getAccordManagedKeyspaces();
     }
 
+    public List<String> getAccordManagedTables()
+    {
+        return ssProxy.getAccordManagedTables();
+    }
+
     public String getClusterName()
     {
         return ssProxy.getClusterName();

src/java/org/apache/cassandra/tools/NodeTool.java

@@ -498,7 +498,7 @@ public class NodeTool
                 else if (defaultKeyspaceSet == KeyspaceSet.NON_SYSTEM)
                     keyspaces.addAll(keyspaces = nodeProbe.getNonSystemKeyspaces());
                 else if (defaultKeyspaceSet == KeyspaceSet.ACCORD_MANAGED)
-                    keyspaces.addAll(nodeProbe.getAccordManagedKeyspace());
+                    keyspaces.addAll(nodeProbe.getAccordManagedKeyspaces());
                 else
                     keyspaces.addAll(nodeProbe.getKeyspaces());
             }

test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java

@@ -27,6 +27,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.google.common.util.concurrent.FutureCallback;
+import org.apache.cassandra.distributed.test.accord.AccordTestBase;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -54,7 +55,6 @@ import org.apache.cassandra.distributed.api.TokenSupplier;
 import org.apache.cassandra.distributed.shared.NetworkTopology;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.reads.repair.BlockingReadRepair;
 import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
 import org.apache.cassandra.utils.concurrent.Condition;
@@ -108,9 +108,9 @@ public class ReadRepairTest extends TestBaseImpl
     {
         try (Cluster cluster = init(Cluster.create(3, config -> config.set("non_serial_write_strategy", brrThroughAccord ? "migration" : "normal"))))
         {
-            cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE));
             cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY (k, c)) " +
                                               String.format("WITH read_repair='%s'", strategy)));
+            AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, "t");
 
             Object[] row = row(1, 1, 1);
             String insertQuery = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)");

test/distributed/org/apache/cassandra/distributed/test/ReadSpeculationTest.java

@@ -25,7 +25,6 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -54,6 +53,7 @@ import org.apache.cassandra.locator.ReplicaCollection;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.service.CassandraDaemon;
+import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.transport.Dispatcher;
 
 import static net.bytebuddy.matcher.ElementMatchers.named;
@@ -86,7 +86,7 @@ public class ReadSpeculationTest extends TestBaseImpl
                 Keyspace keyspace = Keyspace.openIfExists(KEYSPACE);
                 ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(TABLE);
                 DecoratedKey dk = cfs.decorateKey(bytes(PK_VALUE));
-                ReplicaPlan.ForTokenRead plan = ReplicaPlans.forRead(keyspace, dk.getToken(), null,
+                ReplicaPlan.ForTokenRead plan = ReplicaPlans.forRead(keyspace, cfs.getTableId(), dk.getToken(), null,
                                                                      QUORUM, cfs.metadata().params.speculativeRetry,
                                                                      ReadCoordinator.DEFAULT);
                 return plan.contacts().endpointList().stream().map(InetSocketAddress::getAddress).collect(Collectors.toList());

test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java

@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.IntStream;
 
+import org.apache.cassandra.distributed.test.accord.AccordTestBase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -44,7 +45,6 @@ import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
 import org.apache.cassandra.distributed.shared.AssertUtils;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 
@@ -111,7 +111,6 @@ public class ShortReadProtectionTest extends TestBaseImpl
                               .withNodes(NUM_NODES)
                               .withConfig(config -> config.set("hinted_handoff_enabled", false))
                               .start());
-        cluster.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE));
     }
 
     @AfterClass
@@ -437,6 +436,7 @@ public class ShortReadProtectionTest extends TestBaseImpl
 
         private final ConsistencyLevel readConsistencyLevel;
         private final boolean flush, paging;
+        private final String table;
         private final String qualifiedTableName;
 
         private boolean flushed = false;
@@ -446,7 +446,8 @@ public class ShortReadProtectionTest extends TestBaseImpl
             this.readConsistencyLevel = readConsistencyLevel;
             this.flush = flush;
             this.paging = paging;
-            qualifiedTableName = KEYSPACE + ".t_" + seqNumber.getAndIncrement();
+            this.table = "t_" + seqNumber.getAndIncrement();
+            qualifiedTableName = KEYSPACE + '.' + table;
 
             assert readConsistencyLevel == ALL || readConsistencyLevel == QUORUM || readConsistencyLevel == SERIAL
             : "Only ALL and QUORUM consistency levels are supported";
@@ -455,6 +456,7 @@ public class ShortReadProtectionTest extends TestBaseImpl
         private Tester createTable(String query)
         {
             cluster.schemaChange(format(query) + " WITH read_repair='NONE'");
+            AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, table);
             return this;
         }
 

test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java

@@ -78,7 +78,7 @@ public class AccordBootstrapTest extends TestBaseImpl
     private static PartitionKey pk(int key, String keyspace, String table)
     {
         TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
-        return new PartitionKey(keyspace, tid, dk(key));
+        return new PartitionKey(tid, dk(key));
     }
 
     protected void bootstrapAndJoinNode(Cluster cluster)
@@ -451,7 +451,7 @@ public class AccordBootstrapTest extends TestBaseImpl
                             Assert.assertEquals(key, row.getInt("c"));
                             Assert.assertEquals(key, row.getInt("v"));
 
-                            PartitionKey partitionKey = new PartitionKey("ks", tableId, dk);
+                            PartitionKey partitionKey = new PartitionKey(tableId, dk);
 
                             awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(PreLoadContext.contextFor(partitionKey),
                                                                                           partitionKey.toUnseekable(), moveMax, moveMax,

test/distributed/org/apache/cassandra/distributed/test/accord/AccordIntegrationTest.java

@@ -59,10 +59,10 @@ public class AccordIntegrationTest extends AccordTestBase
             IMessageFilters.Filter lostCommit = cluster.filters().verbs(Verb.ACCORD_COMMIT_REQ.id).to(2).drop();
 
             String query = "BEGIN TRANSACTION\n" +
-                           "  LET row1 = (SELECT v FROM " + currentTable + " WHERE k=0 AND c=0);\n" +
+                           "  LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
                            "  SELECT row1.v;\n" +
                            "  IF row1 IS NULL THEN\n" +
-                           "    INSERT INTO " + currentTable + " (k, c, v) VALUES (0, 0, 1);\n" +
+                           "    INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
                            "  END IF\n" +
                            "COMMIT TRANSACTION";
             // row1.v shouldn't have existed when the txn's SELECT was executed
@@ -73,24 +73,24 @@ public class AccordIntegrationTest extends AccordTestBase
 
             // Querying again should trigger recovery...
             query = "BEGIN TRANSACTION\n" +
-                    "  LET row1 = (SELECT v FROM " + currentTable + " WHERE k=0 AND c=0);\n" +
+                    "  LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
                     "  SELECT row1.v;\n" +
                     "  IF row1.v = 1 THEN\n" +
-                    "    UPDATE " + currentTable + " SET v=2 WHERE k = 0 AND c = 0;\n" +
+                    "    UPDATE " + qualifiedTableName + " SET v=2 WHERE k = 0 AND c = 0;\n" +
                     "  END IF\n" +
                     "COMMIT TRANSACTION";
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 1 }, query);
 
             String check = "BEGIN TRANSACTION\n" +
-                    "  SELECT * FROM " + currentTable + " WHERE k = ? AND c = ?;\n" +
+                    "  SELECT * FROM " + qualifiedTableName + " WHERE k = ? AND c = ?;\n" +
                     "COMMIT TRANSACTION";
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] {0, 0, 2}, check, 0, 0);
 
             query = "BEGIN TRANSACTION\n" +
-                    "  LET row1 = (SELECT v FROM " + currentTable + " WHERE k=0 AND c=0);\n" +
+                    "  LET row1 = (SELECT v FROM " + qualifiedTableName + " WHERE k=0 AND c=0);\n" +
                     "  SELECT row1.v;\n" +
                     "  IF row1 IS NULL THEN\n" +
-                    "    INSERT INTO " + currentTable + " (k, c, v) VALUES (0, 0, 3);\n" +
+                    "    INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 3);\n" +
                     "  END IF\n" +
                     "COMMIT TRANSACTION";
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 2 }, query);
@@ -113,16 +113,16 @@ public class AccordIntegrationTest extends AccordTestBase
             })).drop();
 
             String query = "BEGIN TRANSACTION\n" +
-                           "  LET row1 = (SELECT * FROM " + currentTable + " WHERE k = 0 AND c = 0);\n" +
+                           "  LET row1 = (SELECT * FROM " + qualifiedTableName + " WHERE k = 0 AND c = 0);\n" +
                            "  SELECT row1.v;\n" +
                            "  IF row1 IS NULL THEN\n" +
-                           "    INSERT INTO " + currentTable + " (k, c, v) VALUES (0, 0, 1);\n" +
+                           "    INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 1);\n" +
                            "  END IF\n" +
                            "COMMIT TRANSACTION";
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] { null }, query);
 
             String check = "BEGIN TRANSACTION\n" +
-                           "  SELECT * FROM " + currentTable + " WHERE k = ? AND c = ?;\n" +
+                           "  SELECT * FROM " + qualifiedTableName + " WHERE k = ? AND c = ?;\n" +
                            "COMMIT TRANSACTION";
             assertRowEqualsWithPreemptedRetry(cluster, new Object[] { 0, 0, 1 }, check, 0, 0);
         });

test/distributed/org/apache/cassandra/distributed/test/accord/AccordInteroperabilityTest.java

@@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.api.ICoordinator;
 import org.apache.cassandra.distributed.shared.AssertUtils;
-import org.apache.cassandra.service.accord.AccordService;
 
 public class AccordInteroperabilityTest extends AccordTestBase
 {
@@ -45,21 +44,20 @@ public class AccordInteroperabilityTest extends AccordTestBase
     {
         AccordTestBase.setupCluster(builder -> builder.appendConfig(config -> config.set("lwt_strategy", "accord")
                                                                                     .set("non_serial_write_strategy", "accord")), 3);
-        SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE));
     }
 
     @Test
     public void testSerialReadDescending() throws Throwable
     {
-        test("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY(k, c))",
+        test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY(k, c))",
              cluster -> {
                  ICoordinator coordinator = cluster.coordinator(1);
                  for (int i = 1; i <= 10; i++)
-                     coordinator.execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10);
-                 assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100));
-                 assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90));
-                 assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80));
-                 assertRowSerial(cluster, "SELECT c, v FROM " + currentTable + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70));
+                     coordinator.execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, ?, ?) USING TIMESTAMP 0;", ConsistencyLevel.ALL, i, i * 10);
+                 assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 1", AssertUtils.row(10, 100));
+                 assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 2", AssertUtils.row(10, 100), AssertUtils.row(9, 90));
+                 assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 3", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80));
+                 assertRowSerial(cluster, "SELECT c, v FROM " + qualifiedTableName + " WHERE k=0 ORDER BY c DESC LIMIT 4", AssertUtils.row(10, 100), AssertUtils.row(9, 90), AssertUtils.row(8, 80), AssertUtils.row(7, 70));
              }
          );
     }

test/distributed/org/apache/cassandra/distributed/test/accord/AccordMetricsTest.java

@@ -69,16 +69,16 @@ public class AccordMetricsTest extends AccordTestBase
     String writeCql()
     {
         return "BEGIN TRANSACTION\n" +
-               "  LET val = (SELECT v FROM " + currentTable + " WHERE k=? AND c=?);\n" +
+               "  LET val = (SELECT v FROM " + qualifiedTableName + " WHERE k=? AND c=?);\n" +
                "  SELECT val.v;\n" +
-               "  UPDATE " + currentTable + " SET v = v + 1 WHERE k=? AND c=?;\n" +
+               "  UPDATE " + qualifiedTableName + " SET v = v + 1 WHERE k=? AND c=?;\n" +
                "COMMIT TRANSACTION";
     }
 
     String readCql()
     {
         return "BEGIN TRANSACTION\n" +
-               "  LET val = (SELECT v FROM " + currentTable + " WHERE k=? AND c=?);\n" +
+               "  LET val = (SELECT v FROM " + qualifiedTableName + " WHERE k=? AND c=?);\n" +
                "  SELECT val.v;\n" +
                "COMMIT TRANSACTION";
     }
@@ -89,8 +89,8 @@ public class AccordMetricsTest extends AccordTestBase
     public void beforeTest()
     {
         SHARED_CLUSTER.filters().reset();
-        SHARED_CLUSTER.schemaChange("CREATE TABLE " + currentTable + " (k int, c int, v int, PRIMARY KEY (k, c))");
-        SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + currentTable + " (k, c, v) VALUES (0, 0, 0)", ConsistencyLevel.ALL);
+        SHARED_CLUSTER.schemaChange("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, PRIMARY KEY (k, c))");
+        SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + qualifiedTableName + " (k, c, v) VALUES (0, 0, 0)", ConsistencyLevel.ALL);
     }
 
     @Test

test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationTest.java

@@ -61,7 +61,6 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState;
 import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter;
 import org.apache.cassandra.service.consensus.migration.ConsensusTableMigrationState;
@@ -161,7 +160,6 @@ public class AccordMigrationTest extends AccordTestBase
         upperMidToken = partitioner.midpoint(midToken, maxToken);
         lowerMidToken = partitioner.midpoint(minToken, midToken);
         coordinator = SHARED_CLUSTER.coordinator(1);
-        SHARED_CLUSTER.get(1).runOnInstance(() -> AccordService.instance().ensureKeyspaceIsAccordManaged(KEYSPACE));
     }
 
     @AfterClass
@@ -343,13 +341,13 @@ public class AccordMigrationTest extends AccordTestBase
     @Test
     public void testPaxosToAccordCAS() throws Exception
     {
-        test(format(TABLE_FMT, currentTable),
+        test(format(TABLE_FMT, qualifiedTableName),
           cluster -> {
-              String casCQL = format(CAS_FMT, currentTable, CLUSTERING_VALUE);
+              String casCQL = format(CAS_FMT, qualifiedTableName, CLUSTERING_VALUE);
               Consumer<Integer> runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key);
               Consumer<Integer> runCasApplies = key -> assertRowEquals(cluster, new Object[]{true}, casCQL, key);
               Consumer<Integer> runCasOnSecondNode = key -> assertEquals( "[applied]", cluster.coordinator(2).executeWithResult(casCQL, ANY, key).names().get(0));
-              String tableName = currentTable.split("\\.")[1];
+              String tableName = qualifiedTableName.split("\\.")[1];
               int migratingKey = getKeyBetweenTokens(midToken, maxToken);
               int notMigratingKey = getKeyBetweenTokens(minToken, midToken);
               Range<Token> migratingRange = new Range(midToken, maxToken);
@@ -420,7 +418,7 @@ public class AccordMigrationTest extends AccordTestBase
               // Update inserted row so the condition can apply, if the condition check doesn't apply
               // then it won't get to propose/accept
               migratingKey = testingKeys.next();
-              Consumer<Integer> makeCASApply = key -> cluster.coordinator(1).execute("UPDATE " + currentTable + " SET v = 42 WHERE id = ? AND c = ?", ALL, key, CLUSTERING_VALUE);
+              Consumer<Integer> makeCASApply = key -> cluster.coordinator(1).execute("UPDATE " + qualifiedTableName + " SET v = 42 WHERE id = ? AND c = ?", ALL, key, CLUSTERING_VALUE);
               makeCASApply.accept(migratingKey);
               assertTargetAccordWrite(runCasApplies, 1, migratingKey, 1, 1, 1, 0, 1);
 
@@ -469,10 +467,10 @@ public class AccordMigrationTest extends AccordTestBase
     @Test
     public void testPaxosToAccordSerialRead() throws Exception
     {
-        test(format(TABLE_FMT, currentTable),
+        test(format(TABLE_FMT, qualifiedTableName),
           cluster -> {
-              String tableName = currentTable.split("\\.")[1];
-              String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", currentTable, CLUSTERING_VALUE);
+              String tableName = qualifiedTableName.split("\\.")[1];
+              String readCQL = format("SELECT * FROM %s WHERE id = ? and c = %s", qualifiedTableName, CLUSTERING_VALUE);
               Function<Integer, Object[][]> runRead = key -> cluster.coordinator(1).execute(readCQL, SERIAL, key);
               Range<Token> migratingRange = new Range<>(new LongToken(Long.MIN_VALUE + 1), new LongToken(Long.MIN_VALUE));
               List<Range<Token>> migratingRanges = ImmutableList.of(migratingRange);
@@ -495,11 +493,11 @@ public class AccordMigrationTest extends AccordTestBase
     @Test
     public void testAccordToPaxos() throws Exception
     {
-        test(format(TABLE_FMT, currentTable),
+        test(format(TABLE_FMT, qualifiedTableName),
              cluster -> {
-                 String casCQL = format(CAS_FMT, currentTable, CLUSTERING_VALUE);
+                 String casCQL = format(CAS_FMT, qualifiedTableName, CLUSTERING_VALUE);
                  Consumer<Integer> runCasNoApply = key -> assertRowEquals(cluster, new Object[]{false}, casCQL, key);
-                 String tableName = currentTable.split("\\.")[1];
+                 String tableName = qualifiedTableName.split("\\.")[1];
 
                  // Mark a subrange as migrating and finish migrating half of it
                  nodetool(coordinator, "consensus_admin", "begin-migration", "-st", midToken.toString(), "-et", maxToken.toString(), "-tp", "accord", KEYSPACE, tableName);

test/distributed/org/apache/cassandra/distributed/test/accord/AccordSimpleFastPathTest.java

@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.accord;
+
+import java.net.UnknownHostException;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.cassandra.service.accord.AccordFastPath;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.Assert;
+import org.junit.Test;
+
+import accord.local.Node;
+import accord.topology.Topology;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.accord.AccordConfigurationService;
+import org.apache.cassandra.service.accord.AccordService;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AccordSimpleFastPathTest extends TestBaseImpl
+{
+    private static final Logger logger = LoggerFactory.getLogger(AccordSimpleFastPathTest.class);
+
+    private static Node.Id id(int i)
+    {
+        return new Node.Id(i);
+    }
+
+    private static Set<Node.Id> idSet(int... ids)
+    {
+        Set<Node.Id> result = new HashSet<>();
+        for (int id: ids)
+            result.add(id(id));
+        return result;
+    }
+
+    private static InetAddressAndPort ep(int i)
+    {
+        try
+        {
+            return InetAddressAndPort.getByName(String.format("127.0.0.%s:7012", i));
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static Set<InetAddressAndPort> epSet(int... eps)
+    {
+        Set<InetAddressAndPort> result = new HashSet<>();
+        for (int ep: eps)
+            result.add(ep(ep));
+        return result;
+    }
+
+    @Test
+    public void downNodesRemovedFromFastPath() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.build(3)
+                                           .withoutVNodes()
+                                           .withConfig(c -> c.with(Feature.NETWORK).set("accord.enabled", "true"))
+                                           .start()))
+        {
+            cluster.schemaChange("CREATE KEYSPACE ks WITH replication={'class':'SimpleStrategy', 'replication_factor': 3}");
+            cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c))");
+            String query = "BEGIN TRANSACTION\n" +
+                           "  SELECT * FROM ks.tbl WHERE k=0 AND c=0;\n" +
+                           "COMMIT TRANSACTION";
+            cluster.coordinator(1).executeWithResult(query, ConsistencyLevel.ANY);
+
+            InetAddressAndPort node1Addr = InetAddressAndPort.getByAddress(cluster.get(1).broadcastAddress());
+            InetAddressAndPort node2Addr = InetAddressAndPort.getByAddress(cluster.get(2).broadcastAddress());
+            InetAddressAndPort node3Addr = InetAddressAndPort.getByAddress(cluster.get(3).broadcastAddress());
+            int node3Id = cluster.get(3).callOnInstance(() -> ClusterMetadata.current().directory.peerId(FBUtilities.getBroadcastAddressAndPort()).id());
+            long preShutDownEpoch = cluster.stream().map(ii -> ii.callOnInstance(() -> {
+                ClusterMetadata cm = ClusterMetadata.current();
+                AccordFastPath accordFastPath = cm.accordFastPath;
+                Assert.assertEquals(idSet(), accordFastPath.unavailableIds());
+
+                long epoch = cm.epoch.getEpoch();
+                AccordConfigurationService configService = ((AccordService) AccordService.instance()).configurationService();
+                Topology topology = configService.getTopologyForEpoch(epoch);
+                Assert.assertFalse(topology.shards().isEmpty());
+                topology.shards().forEach(shard -> Assert.assertEquals(idSet(1, 2, 3), shard.fastPathElectorate));
+                return cm.epoch.getEpoch();
+            })).max(Comparator.naturalOrder()).get();
+
+            cluster.get(1).runOnInstance(() -> {
+                FailureDetector.instance.forceConviction(InetAddressAndPort.getByAddress(node3Addr));
+                // update is performed in another thread, wait for it to be applied locally before returning
+                for (int i=0; i<10; i++)
+                {
+                    if (ClusterMetadata.current().epoch.getEpoch() == preShutDownEpoch)
+                        FBUtilities.sleepQuietly(100);
+                    else
+                        break;
+                }
+                assert ClusterMetadata.current().epoch.getEpoch() > preShutDownEpoch;
+            });
+
+            cluster.get(1, 2).forEach(ii -> {
+                logger.info("Checking instance {} -> {}", ii, ii.broadcastAddress());
+                ii.runOnInstance(() -> {
+                    ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(preShutDownEpoch + 1));
+                    ClusterMetadata cm = ClusterMetadata.current();
+                    AccordFastPath accordFastPath = cm.accordFastPath;
+                    Assert.assertEquals(preShutDownEpoch + 1, cm.epoch.getEpoch());
+                    Assert.assertEquals(idSet(node3Id), accordFastPath.unavailableIds());
+                });
+
+            }
+            );
+
+            // confirm a duplicate conviction doesn't create a new epoch
+            cluster.get(2).runOnInstance(() -> {
+                FailureDetector.instance.forceConviction(InetAddressAndPort.getByAddress(node3Addr));
+            });
+
+            cluster.get(1, 2).forEach(ii -> ii.runOnInstance(() -> {
+                ClusterMetadata cm = ClusterMetadata.current();
+                Assert.assertEquals(preShutDownEpoch + 1, cm.epoch.getEpoch());
+            }));
+        }
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java

@@ -30,6 +30,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
+import accord.coordinate.Invalidated;
 import com.google.common.base.Splitter;
 import com.google.common.primitives.Ints;
 import org.junit.After;
@@ -85,7 +86,8 @@ public abstract class AccordTestBase extends TestBaseImpl
 
     protected static Cluster SHARED_CLUSTER;
 
-    protected String currentTable;
+    protected String tableName;
+    protected String qualifiedTableName;
 
     public static void setupCluster(Function<Builder, Builder> options, int nodes) throws IOException
     {
@@ -102,7 +104,8 @@ public abstract class AccordTestBase extends TestBaseImpl
     @Before
     public void setup()
     {
-        currentTable = KEYSPACE + ".tbl" + COUNTER.getAndIncrement();
+        tableName = "tbl" + COUNTER.getAndIncrement();
+        qualifiedTableName = KEYSPACE + '.' + tableName;
     }
 
     @After
@@ -129,11 +132,17 @@ public abstract class AccordTestBase extends TestBaseImpl
         test(Collections.singletonList(tableDDL), fn);
     }
 
+    public static void ensureTableIsAccordManaged(Cluster cluster, String ksname, String tableName)
+    {
+        cluster.get(1).runOnInstance(() -> AccordService.instance().ensureTableIsAccordManaged(ksname, tableName));
+    }
+
     protected void test(List<String> ddls, FailingConsumer<Cluster> fn) throws Exception
     {
         for (String ddl : ddls)
             SHARED_CLUSTER.schemaChange(ddl);
 
+        ensureTableIsAccordManaged(SHARED_CLUSTER, KEYSPACE, tableName);
         // Evict commands from the cache immediately to expose problems loading from disk.
         SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0)));
 
@@ -149,7 +158,7 @@ public abstract class AccordTestBase extends TestBaseImpl
 
     protected void test(FailingConsumer<Cluster> fn) throws Exception
     {
-        test("CREATE TABLE " + currentTable + " (k int, c int, v int, primary key (k, c))", fn);
+        test("CREATE TABLE " + qualifiedTableName + " (k int, c int, v int, primary key (k, c))", fn);
     }
 
     protected static ConsensusMigrationState getMigrationStateSnapshot(IInvokableInstance instance) throws IOException
@@ -331,6 +340,12 @@ public abstract class AccordTestBase extends TestBaseImpl
         return result;
     }
 
+    private static boolean hasRootCause(RuntimeException ex, Class<? extends RuntimeException> klass)
+    {
+        return AssertionUtils.rootCauseIs(klass).matches(ex);
+
+    }
+
     private static SimpleQueryResult executeWithRetry0(int count, Cluster cluster, String check, Object... boundValues)
     {
         try
@@ -339,7 +354,7 @@ public abstract class AccordTestBase extends TestBaseImpl
         }
         catch (RuntimeException ex)
         {
-            if (count <= MAX_RETRIES && (AssertionUtils.rootCauseIs(ReadPreemptedException.class).matches(ex) || AssertionUtils.rootCauseIs(WritePreemptedException.class).matches(ex)))
+            if (count <= MAX_RETRIES && (hasRootCause(ex, ReadPreemptedException.class) || hasRootCause(ex, WritePreemptedException.class) || hasRootCause(ex, Invalidated.class)))
             {
                 logger.warn("[Retry attempt={}] Preempted failure for\n{}", count, check);
                 return executeWithRetry0(count + 1, cluster, check, boundValues);

test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java

@@ -63,12 +63,13 @@ import org.apache.cassandra.tcm.RegistrationStatus;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.log.LocalLog;
 import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.service.accord.AccordFastPath;
 import org.apache.cassandra.tcm.membership.Location;
 import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.AccordKeyspaces;
+import org.apache.cassandra.tcm.ownership.AccordTables;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.TokenMap;
 import org.apache.cassandra.tcm.ownership.UniformRangePlacement;
@@ -149,7 +150,8 @@ public class ClusterMetadataTestHelper
                                    Directory.EMPTY,
                                    new TokenMap(partitioner),
                                    DataPlacements.empty(),
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    LockedRanges.EMPTY,
                                    InProgressSequences.EMPTY,
                                    null,
@@ -164,7 +166,8 @@ public class ClusterMetadataTestHelper
                                    null,
                                    null,
                                    DataPlacements.empty(),
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    null,
                                    null,
                                    null,
@@ -179,7 +182,8 @@ public class ClusterMetadataTestHelper
                                    null,
                                    null,
                                    DataPlacements.empty(),
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    null,
                                    null,
                                    null,

test/simulator/test/org/apache/cassandra/simulator/test/AccordJournalSimulationTest.java

@@ -207,7 +207,7 @@ public class AccordJournalSimulationTest extends SimulationTestBase
         private static TxnRequest<?> toRequest(int event)
         {
             TxnId id = toTxnId(event);
-            Ranges ranges = Ranges.of(new TokenRange(AccordRoutingKey.SentinelKey.min("system"), AccordRoutingKey.SentinelKey.max("system")));
+            Ranges ranges = Ranges.of(new TokenRange(AccordRoutingKey.SentinelKey.min(tableId), AccordRoutingKey.SentinelKey.max(tableId)));
             Topologies topologies = Utils.topologies(TopologyUtils.initialTopology(new Node.Id[] {node}, ranges, 3));
             Keys keys = Keys.of(toKey(0));
             Txn txn = new Txn.InMemory(keys, new TxnRead(new TxnNamedRead[0], keys, null), TxnQuery.ALL, new NoopUpdate());
@@ -222,7 +222,7 @@ public class AccordJournalSimulationTest extends SimulationTestBase
 
         private static PartitionKey toKey(int a)
         {
-            return new PartitionKey(KEYSPACE, tableId, Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(a)));
+            return new PartitionKey(tableId, Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(a)));
         }
 
         private static final TableId tableId = TableId.fromUUID(new UUID(0, 0));
@@ -233,7 +233,7 @@ public class AccordJournalSimulationTest extends SimulationTestBase
             return new FullKeyRoute(key, true, new RoutingKey[]{ key });
         }
 
-        private static final RoutingKey key = new AccordRoutingKey.TokenKey("system", new Murmur3Partitioner.LongToken(42));
+        private static final RoutingKey key = new AccordRoutingKey.TokenKey(tableId, new Murmur3Partitioner.LongToken(42));
     }
 
     public static class NoopUpdate implements Update

test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java

@@ -291,11 +291,11 @@ public class DescribeStatementTest extends CQLTester
             row(KEYSPACE, "keyspace", KEYSPACE,
                 "CREATE KEYSPACE " + KEYSPACE +
                 " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
-                "  AND durable_writes = true;"),
+                "  AND durable_writes = true  AND fast_path = 'simple';"),
             row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST,
                 "CREATE KEYSPACE " + KEYSPACE_PER_TEST +
                 " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
-                "  AND durable_writes = true;"),
+                "  AND durable_writes = true  AND fast_path = 'simple';"),
             row("test", "keyspace", "test", keyspaceOutput()),
             row("test", "table", "has_all_types", allTypesTable()),
             row("test", "table", "\"Test\"", testTableOutput()),
@@ -697,7 +697,8 @@ public class DescribeStatementTest extends CQLTester
             assertRowsNet(executeDescribeNet(KEYSPACE_PER_TEST, "DESCRIBE KEYSPACE " + KEYSPACE_PER_TEST),
                           row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST, "CREATE KEYSPACE " + KEYSPACE_PER_TEST +
                                                                                 " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
-                                                                                "  AND durable_writes = true;"),
+                                                                                "  AND durable_writes = true" +
+                                                                                "  AND fast_path = 'simple';"),
                           row(KEYSPACE_PER_TEST, "type", type2, "CREATE TYPE " + KEYSPACE_PER_TEST + "." + type2 + " (\n" +
                                                                 "    x text,\n" +
                                                                 "    y text\n" +
@@ -802,7 +803,8 @@ public class DescribeStatementTest extends CQLTester
 
         String expectedKeyspaceStmt = "CREATE KEYSPACE " + KEYSPACE_PER_TEST +
                                       " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" +
-                                      "  AND durable_writes = true;";
+                                      "  AND durable_writes = true" +
+                                      "  AND fast_path = 'simple';";
 
         String expectedTableStmt = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" +
                                    "    id int PRIMARY KEY,\n" +
@@ -1130,6 +1132,7 @@ public class DescribeStatementTest extends CQLTester
                "    AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" +
                "    AND memtable = 'default'\n" +
                "    AND crc_check_chance = 1.0\n" +
+               "    AND fast_path = 'keyspace'\n" +
                "    AND default_time_to_live = 0\n" +
                "    AND extensions = {}\n" +
                "    AND gc_grace_seconds = 864000\n" +
@@ -1170,7 +1173,7 @@ public class DescribeStatementTest extends CQLTester
 
     private static String keyspaceOutput()
     {
-        return "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;";
+        return "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true  AND fast_path = 'simple';";
     }
 
     private void describeError(String cql, String msg) throws Throwable

test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java

@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.io.Files;
 
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -309,6 +310,7 @@ public class SchemaCQLHelperTest extends CQLTester
                .compaction(CompactionParams.lcs(Collections.singletonMap("sstable_size_in_mb", "1")))
                .compression(CompressionParams.lz4(1 << 16, 1 << 15))
                .crcCheckChance(0.3)
+               .fastPath(FastPathStrategy.simple())
                .defaultTimeToLive(4)
                .gcGraceSeconds(5)
                .minIndexInterval(6)
@@ -336,6 +338,7 @@ public class SchemaCQLHelperTest extends CQLTester
                             "    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor', 'min_compress_ratio': '2.0'}\n" +
                             "    AND memtable = 'default'\n" +
                             "    AND crc_check_chance = 0.3\n" +
+                            "    AND fast_path = 'simple'\n" +
                             "    AND default_time_to_live = 4\n" +
                             "    AND extensions = {'ext1': 0x76616c31}\n" +
                             "    AND gc_grace_seconds = 5\n" +

test/unit/org/apache/cassandra/dht/BootStrapperTest.java

@@ -212,7 +212,8 @@ public class BootStrapperTest extends CassandraTestBase
                false,
                1,
                movements.left,
-               movements.right);
+               movements.right,
+               true);
     }
 
     private boolean includesWraparound(Collection<Range<Token>> toFetch)

test/unit/org/apache/cassandra/dht/PartitionerTestCase.java

@@ -37,6 +37,7 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.accord.TokenRange;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 
+import static org.apache.cassandra.service.accord.AccordTestUtils.TABLE_ID1;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -288,9 +289,9 @@ public abstract class PartitionerTestCase
         if (less.equals(more) && less.isMinimum())
             ranges = Ranges.EMPTY;
         else if (less.equals(more))
-            ranges = Ranges.of(new TokenRange(new TokenKey("", partitioner.getMinimumToken()), new TokenKey("", less)));
+            ranges = Ranges.of(new TokenRange(new TokenKey(TABLE_ID1, partitioner.getMinimumToken()), new TokenKey(TABLE_ID1, less)));
         else
-            ranges = Ranges.of(new TokenRange(new TokenKey("", less), new TokenKey("", more)));
+            ranges = Ranges.of(new TokenRange(new TokenKey(TABLE_ID1, less), new TokenKey(TABLE_ID1, more)));
 
         AccordSplitter splitter = partitioner.accordSplitter().apply(ranges);
         BigInteger lv = splitter.valueForToken(less);
@@ -303,11 +304,11 @@ public abstract class PartitionerTestCase
 
     void testSplitter(Token start, Token end)
     {
-        accord.primitives.Range range = new TokenRange(new TokenKey("", start), new TokenKey("", end));
+        accord.primitives.Range range = new TokenRange(new TokenKey(TABLE_ID1, start), new TokenKey(TABLE_ID1, end));
         AccordSplitter splitter = partitioner.accordSplitter().apply(Ranges.of(range));
         if (!start.isMinimum())
-            testSplitter(new TokenRange(new TokenKey("", partitioner.getMinimumToken()), new TokenKey("", start)));
-        testSplitter(new TokenRange(new TokenKey("", start), new TokenKey("", splitter.tokenForValue(splitter.maximumValue()))));
+            testSplitter(new TokenRange(new TokenKey(TABLE_ID1, partitioner.getMinimumToken()), new TokenKey(TABLE_ID1, start)));
+        testSplitter(new TokenRange(new TokenKey(TABLE_ID1, start), new TokenKey(TABLE_ID1, splitter.tokenForValue(splitter.maximumValue()))));
         checkRoundTrip(start, splitter.tokenForValue(splitter.valueForToken(start)));
         checkRoundTrip(end, splitter.tokenForValue(splitter.valueForToken(end)));
     }

test/unit/org/apache/cassandra/locator/AssureSufficientLiveNodesTest.java

@@ -32,6 +32,7 @@ import java.util.function.Supplier;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.schema.*;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -45,10 +46,6 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper;
 import org.apache.cassandra.exceptions.UnavailableException;
-import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.SchemaTestUtil;
-import org.apache.cassandra.schema.Tables;
 import org.apache.cassandra.service.reads.NeverSpeculativeRetryPolicy;
 import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.utils.FBUtilities;
@@ -82,6 +79,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
     private static final String DC3 = "datacenter3";
     private static final int RACE_TEST_LOOPS = 100;
     private static final Token tk = new Murmur3Partitioner.LongToken(0);
+    private static final TableId TABLE_ID = TableId.generate();
 
     @BeforeClass
     public static void setUpClass() throws Throwable
@@ -140,7 +138,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
             // alter to
             KeyspaceParams.nts(DC1, 3, DC2, 3),
             // test
-            keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
+            keyspace -> ReplicaPlans.forRead(keyspace, TABLE_ID, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
         );
     }
 
@@ -173,7 +171,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
             // alter to
             KeyspaceParams.nts(DC1, 3, DC2, 3),
             // test
-            keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
+            keyspace -> ReplicaPlans.forRead(keyspace, TABLE_ID, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
         );
         raceOfReplicationStrategyTest(
             // init. The # of live endpoints is 3 = 2 + 1
@@ -181,7 +179,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
             // alter to. (3 + 3) / 2 + 1 > 3
             KeyspaceParams.nts(DC1, 2, DC2, 1, DC3, 3),
             // test
-            keyspace -> ReplicaPlans.forRead(keyspace, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
+            keyspace -> ReplicaPlans.forRead(keyspace, TABLE_ID, tk, null, QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
         );
     }
 
@@ -205,7 +203,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
             // alter to
             KeyspaceParams.nts(DC1, 3),
             // test
-            keyspace -> ReplicaPlans.forRead(keyspace, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
+            keyspace -> ReplicaPlans.forRead(keyspace, TABLE_ID, tk, null, EACH_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
         );
     }
 
@@ -229,7 +227,7 @@ public class AssureSufficientLiveNodesTest extends CassandraTestBase
             // alter to
             KeyspaceParams.nts(DC1, 3),
             // test
-            keyspace -> ReplicaPlans.forRead(keyspace, tk, null, LOCAL_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
+            keyspace -> ReplicaPlans.forRead(keyspace, TABLE_ID, tk, null, LOCAL_QUORUM, NeverSpeculativeRetryPolicy.INSTANCE, ReadCoordinator.DEFAULT)
         );
     }
 

test/unit/org/apache/cassandra/locator/MetaStrategyTest.java

@@ -26,6 +26,8 @@ import java.util.Map;
 import java.util.Set;
 
 import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.service.accord.AccordFastPath;
+import org.apache.cassandra.tcm.ownership.AccordTables;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -39,7 +41,6 @@ import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.Location;
 import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.AccordKeyspaces;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.TokenMap;
 import org.apache.cassandra.tcm.sequences.InProgressSequences;
@@ -89,7 +90,8 @@ public class MetaStrategyTest
                                    directory,
                                    tokenMap,
                                    DataPlacements.EMPTY,
-                                   AccordKeyspaces.EMPTY,
+                                   AccordTables.EMPTY,
+                                   AccordFastPath.EMPTY,
                                    LockedRanges.EMPTY,
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,

test/unit/org/apache/cassandra/schema/FastPathSchemaTest.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.schema;
+
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
+import org.apache.cassandra.service.accord.fastpath.ParameterizedFastPathStrategy;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.transformations.AddAccordTable;
+
+import static java.lang.String.format;
+
+public class FastPathSchemaTest
+{
+    private static String KEYSPACE = "ks";
+    private static int ksCount = 0;
+
+    @BeforeClass
+    public static void setupClass()
+    {
+        DatabaseDescriptor.daemonInitialization();
+        ServerTestUtils.prepareServer();
+        SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create(KEYSPACE, KeyspaceParams.simple(1), Tables.of()));
+    }
+
+    @Before
+    public void setup()
+    {
+        KEYSPACE = format("ks_%s", ksCount++);
+    }
+
+
+    private static void process(String fmt, Object... objects)
+    {
+        QueryProcessor.process(format(fmt, objects), ConsistencyLevel.ANY);
+    }
+
+    @Test
+    public void keyspaceInheriting()
+    {
+        process("CREATE KEYSPACE %s with replication={'class':'SimpleStrategy', 'replication_factor':1} AND fast_path='simple'", KEYSPACE);
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertSame(FastPathStrategy.simple(), ksm.params.fastPath);
+
+        process("CREATE TABLE %s.tbl (k int primary key, v int)", KEYSPACE);
+        TableMetadata tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl");
+        Assert.assertSame(FastPathStrategy.inheritKeyspace(), tbm.params.fastPath);
+
+        Epoch epoch = ClusterMetadata.current().epoch;
+        AddAccordTable.addTable(tbm.id);
+
+        Assert.assertEquals(epoch.getEpoch() + 1, ClusterMetadata.current().epoch.getEpoch());
+    }
+
+    @Test
+    public void keyspaceModification()
+    {
+        process("CREATE KEYSPACE %s with replication={'class':'SimpleStrategy', 'replication_factor':1} AND fast_path='simple'", KEYSPACE);
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertSame(FastPathStrategy.simple(), ksm.params.fastPath);
+        process("ALTER KEYSPACE %s with fast_path={'size':2, 'dcs':'dc1,dc2'}", KEYSPACE);
+
+        ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertSame(FastPathStrategy.Kind.PARAMETERIZED, ksm.params.fastPath.kind());
+        ParameterizedFastPathStrategy strategy = (ParameterizedFastPathStrategy) ksm.params.fastPath;
+        Assert.assertEquals(2, strategy.size);
+        Assert.assertEquals(Arrays.asList("dc1", "dc2"), strategy.dcStrings());
+    }
+
+    @Test(expected = ConfigurationException.class)
+    public void keyspaceInheritingFailure()
+    {
+        process("CREATE KEYSPACE %s with replication={'class':'SimpleStrategy', 'replication_factor':1} AND fast_path='keyspace'", KEYSPACE);
+    }
+
+    @Test
+    public void tableModification()
+    {
+        process("CREATE KEYSPACE %s with replication={'class':'SimpleStrategy', 'replication_factor':1} AND fast_path='simple'", KEYSPACE);
+        KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(KEYSPACE);
+        Assert.assertSame(FastPathStrategy.simple(), ksm.params.fastPath);
+
+        process("CREATE TABLE %s.tbl (k int primary key, v int)", KEYSPACE);
+        TableMetadata tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl");
+        Assert.assertSame(FastPathStrategy.inheritKeyspace(), tbm.params.fastPath);
+
+        AddAccordTable.addTable(tbm.id);
+
+        process("ALTER TABLE %s.tbl WITH fast_path='simple'", KEYSPACE);
+        tbm = Schema.instance.getTableMetadata(KEYSPACE, "tbl");
+        Assert.assertSame(FastPathStrategy.simple(), tbm.params.fastPath);
+    }
+}

test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java

@@ -61,7 +61,9 @@ import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.accord.AccordCachingState.Modified;
 import org.apache.cassandra.service.accord.api.PartitionKey;
@@ -108,6 +110,7 @@ public class AccordCommandStoreTest
         AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
 
         QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, c, v) VALUES (0, 0, 1)");
+        TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id;
         TxnId oldTxnId1 = txnId(1, clock.incrementAndGet(), 1);
         TxnId oldTxnId2 = txnId(1, clock.incrementAndGet(), 1);
         TxnId oldTimestamp = txnId(1, clock.incrementAndGet(), 1);
@@ -147,7 +150,7 @@ public class AccordCommandStoreTest
 
         Apply apply =
             Apply.SerializationSupport.create(txnId,
-                                              route.slice(Ranges.of(TokenRange.fullRange("ks"))),
+                                              route.slice(Ranges.of(TokenRange.fullRange(tableId))),
                                               1L,
                                               Apply.Kind.Minimal,
                                               depTxn.keys(),

test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java

@@ -81,7 +81,7 @@ public class AccordCommandTest
     private static PartitionKey key(int k)
     {
         TableMetadata metadata = Schema.instance.getTableMetadata("ks", "tbl");
-        return new PartitionKey(metadata.keyspace, metadata.id, metadata.partitioner.decorateKey(ByteBufferUtil.bytes(k)));
+        return new PartitionKey(metadata.id, metadata.partitioner.decorateKey(ByteBufferUtil.bytes(k)));
     }
 
     /**