CEP-15: Accord Bootstrap Integration

Patch by Blake Eggleston and Benedict Elliott Smith; Reviewed by David
Capwell for CASSANDRA-17101

CEP-15: Accord TCM integration

Patch by Blake Eggleston; Reviewed by David Capwell for CASSANDRA-18444

modules/accord

@@ -1 +1 @@
-Subproject commit 8830d97ba517fb2d0f7f22e8e6b886a98839e694
+Subproject commit 03f937175dbcf04243bb0ac48b64746c1a07bc9c

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -236,6 +236,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         ANTICOMPACTION,
         SCHEMA_CHANGE,
         OWNED_RANGES_CHANGE,
+        ACCORD,
         UNIT_TESTS // explicitly requested flush needed for a test
     }
 

src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java

@@ -64,7 +64,6 @@ public class AccordVirtualTables
         public DataSet data()
         {
             IAccordService accord = AccordService.instance();
-            accord.createEpochFromConfigUnsafe();
 
             long epoch = accord.currentEpoch();
 

src/java/org/apache/cassandra/dht/BootStrapper.java

@@ -40,6 +40,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.metrics.StorageMetrics;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.streaming.StreamEvent;
 import org.apache.cassandra.streaming.StreamEventHandler;
 import org.apache.cassandra.streaming.StreamOperation;
@@ -136,6 +137,8 @@ public class BootStrapper extends ProgressEventNotifierSupport
             logger.debug("Schema does not contain any non-local keyspaces to stream on bootstrap");
         for (String keyspaceName : nonLocalStrategyKeyspaces)
         {
+            if (AccordService.instance().isAccordManagedKeyspace(keyspaceName))
+                continue;
             KeyspaceMetadata ksm = metadata.schema.getKeyspaces().get(keyspaceName).get();
             if (ksm.params.replication.isMeta())
                 continue;

src/java/org/apache/cassandra/net/Verb.java

@@ -79,10 +79,12 @@ import org.apache.cassandra.schema.SchemaVersionVerbHandler;
 import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.service.accord.serializers.AcceptSerializers;
 import org.apache.cassandra.service.accord.serializers.ApplySerializers;
+import org.apache.cassandra.service.accord.AccordLocalSyncNotifier;
 import org.apache.cassandra.service.accord.serializers.BeginInvalidationSerializers;
 import org.apache.cassandra.service.accord.serializers.CheckStatusSerializers;
 import org.apache.cassandra.service.accord.serializers.CommitSerializers;
 import org.apache.cassandra.service.accord.serializers.EnumSerializer;
+import org.apache.cassandra.service.accord.serializers.FetchSerializers;
 import org.apache.cassandra.service.accord.serializers.GetDepsSerializers;
 import org.apache.cassandra.service.accord.serializers.InformDurableSerializers;
 import org.apache.cassandra.service.accord.serializers.InformHomeDurableSerializers;
@@ -91,6 +93,8 @@ import org.apache.cassandra.service.accord.serializers.PreacceptSerializers;
 import org.apache.cassandra.service.accord.serializers.ReadDataSerializers;
 import org.apache.cassandra.service.accord.serializers.RecoverySerializers;
 import org.apache.cassandra.service.accord.serializers.WaitOnCommitSerializer;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.Commit.Agreed;
 import org.apache.cassandra.service.paxos.PaxosCommit;
 import org.apache.cassandra.service.paxos.PaxosCommitAndPrepare;
 import org.apache.cassandra.service.paxos.PaxosPrepare;
@@ -117,8 +121,6 @@ import org.apache.cassandra.tcm.serialization.MessageSerializers;
 import org.apache.cassandra.utils.BooleanSerializer;
 import org.apache.cassandra.service.EchoVerbHandler;
 import org.apache.cassandra.service.SnapshotVerbHandler;
-import org.apache.cassandra.service.paxos.Commit;
-import org.apache.cassandra.service.paxos.Commit.Agreed;
 import org.apache.cassandra.service.paxos.PrepareResponse;
 import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
 import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
@@ -290,6 +292,10 @@ public enum Verb
     ACCORD_CHECK_STATUS_REQ         (140, P2, writeTimeout, IMMEDIATE,          () -> CheckStatusSerializers.request,       () -> AccordService.instance().verbHandler(), ACCORD_CHECK_STATUS_RSP       ),
     ACCORD_GET_DEPS_RSP             (143, P2, writeTimeout, REQUEST_RESPONSE,   () -> GetDepsSerializers.reply,             RESPONSE_HANDLER                                                            ),
     ACCORD_GET_DEPS_REQ             (142, P2, writeTimeout, IMMEDIATE,          () -> GetDepsSerializers.request,           () -> AccordService.instance().verbHandler(), ACCORD_GET_DEPS_RSP           ),
+    ACCORD_FETCH_DATA_RSP           (145, P2, writeTimeout, REQUEST_RESPONSE,   () -> FetchSerializers.reply,               RESPONSE_HANDLER                                                            ),
+    ACCORD_FETCH_DATA_REQ           (144, P2, writeTimeout, IMMEDIATE,          () -> FetchSerializers.request,             () -> AccordService.instance().verbHandler(), ACCORD_FETCH_DATA_RSP         ),
+    ACCORD_SYNC_NOTIFY_RSP          (147, P2, writeTimeout, REQUEST_RESPONSE,   () -> AccordLocalSyncNotifier.Acknowledgement.serializer, RESPONSE_HANDLER                                              ),
+    ACCORD_SYNC_NOTIFY_REQ          (146, P2, writeTimeout, IMMEDIATE,          () -> AccordLocalSyncNotifier.Notification.serializer, () -> AccordLocalSyncNotifier.verbHandler, ACCORD_SYNC_NOTIFY_RSP),
 
 
     // generic failure response

src/java/org/apache/cassandra/service/Rebuild.java

@@ -44,11 +44,15 @@ import org.apache.cassandra.locator.RangesAtEndpoint;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamResultFuture;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.MovementMap;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
@@ -119,11 +123,16 @@ public class Rebuild
             if (keyspace == null)
             {
                 for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names())
+                {
+                    if (AccordService.instance().isAccordManagedKeyspace(keyspaceName))
+                        continue;
                     streamer.addKeyspaceToFetch(keyspaceName);
+                }
             }
             else if (tokens == null)
             {
-                streamer.addKeyspaceToFetch(keyspace);
+                if (!AccordService.instance().isAccordManagedKeyspace(keyspace))
+                    streamer.addKeyspaceToFetch(keyspace);
             }
             else
             {
@@ -150,10 +159,16 @@ public class Rebuild
                     streamer.addSourceFilter(new RangeStreamer.AllowedSourcesFilter(sources));
                 }
 
-                streamer.addKeyspaceToFetch(keyspace);
+                if (!AccordService.instance().isAccordManagedKeyspace(keyspace))
+                    streamer.addKeyspaceToFetch(keyspace);
             }
 
-            streamer.fetchAsync().get();
+            StreamResultFuture resultFuture = streamer.fetchAsync();
+            // wait for result
+            Future<Void> accordReady = AccordService.instance().epochReady(metadata.epoch);
+            Future<?> ready = FutureCombiner.allOf(resultFuture, accordReady);
+            // wait for result
+            ready.get();
         }
         catch (InterruptedException e)
         {

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -1884,7 +1884,8 @@ public class StorageProxy implements StorageProxyMBean
     private static PartitionIterator readWithConsensus(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException
     {
-        if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord)
+        // TCM explicitly relies on paxos and doesn't work with accord
+        if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord && !group.metadata().keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME))
         {
             return readWithAccord(group, consistencyLevel);
         }

src/java/org/apache/cassandra/service/accord/AccordCallback.java

@@ -34,17 +34,19 @@ import org.apache.cassandra.net.RequestCallback;
 class AccordCallback<T extends Reply> extends SafeCallback<T> implements RequestCallback<T>
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordCallback.class);
+    private final AccordEndpointMapper endpointMapper;
 
-    public AccordCallback(AgentExecutor executor, Callback<T> callback)
+    public AccordCallback(AgentExecutor executor, Callback<T> callback, AccordEndpointMapper endpointMapper)
     {
         super(executor, callback);
+        this.endpointMapper = endpointMapper;
     }
 
     @Override
     public void onResponse(Message<T> msg)
     {
         logger.debug("Received response {} from {}", msg.payload, msg.from());
-        success(EndpointMapping.endpointToId(msg.from()), msg.payload);
+        success(endpointMapper.mappedId(msg.from()), msg.payload);
     }
 
     private static Throwable convertReason(RequestFailureReason reason)
@@ -59,7 +61,7 @@ class AccordCallback<T extends Reply> extends SafeCallback<T> implements Request
     {
         logger.debug("Received failure {} from {} for {}", failureReason, from, this);
         // TODO (now): we should distinguish timeout failures with some placeholder Exception
-        failure(EndpointMapping.endpointToId(from), convertReason(failureReason));
+        failure(endpointMapper.mappedId(from), convertReason(failureReason));
     }
 
     @Override

src/java/org/apache/cassandra/service/accord/AccordCommandStore.java

@@ -65,6 +65,7 @@ import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import accord.utils.async.Observable;
@@ -150,6 +151,16 @@ public class AccordCommandStore extends CommandStore
                                 this::loadCommandsForKey,
                                 this::saveCommandsForKey,
                                 AccordObjectSizes::commandsForKey);
+        AccordKeyspace.loadCommandStoreMetadata(id, ((rejectBefore, bootstrapBeganAt, safeToRead) -> {
+            executor.submit(() -> {
+                if (rejectBefore != null)
+                    super.setRejectBefore(rejectBefore);
+                if (bootstrapBeganAt != null)
+                    super.setBootstrapBeganAt(bootstrapBeganAt);
+                if (safeToRead != null)
+                    super.setSafeToRead(safeToRead);
+            });
+        }));
         executor.execute(() -> CommandStore.register(this));
         executor.execute(this::loadRangesToCommands);
     }
@@ -437,4 +448,25 @@ public class AccordCommandStore extends CommandStore
     {
         executor.shutdown();
     }
+
+    protected void setRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
+    {
+        super.setRejectBefore(newRejectBefore);
+        // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases
+        AccordKeyspace.updateRejectBefore(this, newRejectBefore);
+    }
+
+    protected void setBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
+    {
+        super.setBootstrapBeganAt(newBootstrapBeganAt);
+        // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases
+        AccordKeyspace.updateBootstrapBeganAt(this, newBootstrapBeganAt);
+    }
+
+    protected void setSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
+    {
+        super.setSafeToRead(newSafeToRead);
+        // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases
+        AccordKeyspace.updateSafeToRead(this, newSafeToRead);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordCommandStores.java

@@ -141,9 +141,9 @@ public class AccordCommandStores extends CommandStores
     }
 
     @Override
-    public synchronized Supplier<EpochReady> updateTopology(Node node, Topology newTopology)
+    public synchronized Supplier<EpochReady> updateTopology(Node node, Topology newTopology, boolean startSync)
     {
-        Supplier<EpochReady> start = super.updateTopology(node, newTopology);
+        Supplier<EpochReady> start = super.updateTopology(node, newTopology, startSync);
         return () -> {
             EpochReady ready = start.get();
             ready.metadata.addCallback(() -> {

src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java

@@ -18,98 +18,385 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
-import javax.annotation.concurrent.GuardedBy;
+import javax.annotation.Nullable;
 
-import accord.api.ConfigurationService;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.impl.AbstractConfigurationService;
 import accord.local.Node;
 import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
+import org.agrona.collections.Long2ObjectHashMap;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.MessageDelivery;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.accord.AccordKeyspace.EpochDiskState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.listeners.ChangeListener;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
 
-/**
- * Currently a stubbed out config service meant to be triggered from a dtest
- */
-public class AccordConfigurationService implements ConfigurationService
+// TODO: listen to FailureDetector and rearrange fast path accordingly
+public class AccordConfigurationService extends AbstractConfigurationService<AccordConfigurationService.EpochState, AccordConfigurationService.EpochHistory> implements ChangeListener, AccordEndpointMapper, AccordLocalSyncNotifier.Listener
 {
-    private final Node.Id localId;
-    private final List<Listener> listeners = new CopyOnWriteArrayList<>();
-    @GuardedBy("this")
-    private final List<Topology> epochs = new ArrayList<>();
+    private static final Logger logger = LoggerFactory.getLogger(AccordConfigurationService.class);
 
-    public AccordConfigurationService(Node.Id localId)
-    {
-        this.localId = localId;
-        epochs.add(Topology.EMPTY);
-    }
+    private final MessageDelivery messagingService;
+    private final IFailureDetector failureDetector;
+    private EpochDiskState diskState = EpochDiskState.EMPTY;
+    private enum State { INITIALIZED, LOADING, STARTED }
 
-    @Override
-    public void registerListener(Listener listener)
-    {
-        listeners.add(listener);
-    }
+    private State state = State.INITIALIZED;
+    private volatile EndpointMapping mapping = EndpointMapping.EMPTY;
+    private final Long2ObjectHashMap<AccordLocalSyncNotifier> syncNotifiers = new Long2ObjectHashMap<>();
 
-    @Override
-    public synchronized Topology currentTopology()
-    {
-        return epochs.get(epochs.size() - 1);
-    }
+    public enum SyncStatus { NOT_STARTED, NOTIFYING, COMPLETED }
 
-    @Override
-    public synchronized Topology getTopologyForEpoch(long epoch)
+    static class EpochState extends AbstractConfigurationService.AbstractEpochState
     {
-        return epochs.get((int) epoch);
-    }
+        SyncStatus syncStatus = SyncStatus.NOT_STARTED;
+        protected final AsyncResult.Settable<Void> localSyncNotified = AsyncResults.settable();
 
-    @Override
-    public synchronized void fetchTopologyForEpoch(long epoch)
-    {
-        Topology current = currentTopology();
-        if (epoch < current.epoch())
-            return;
-        while (current.epoch() < epoch)
+        public EpochState(long epoch)
         {
-            current = AccordTopologyUtils.createTopology(epochs.size());
-            unsafeAddEpoch(current);
+            super(epoch);
+        }
+
+        void setSyncStatus(SyncStatus status)
+        {
+            this.syncStatus = status;
+            if (status == SyncStatus.COMPLETED)
+                localSyncNotified.trySuccess(null);
+        }
+
+        AsyncResult<Topology> received()
+        {
+            return received;
+        }
+
+        AsyncResult<Void> acknowledged()
+        {
+            return acknowledged;
+        }
+
+        @Nullable AsyncResult<Void> reads()
+        {
+            return reads;
+        }
+
+        AsyncResult.Settable<Void> localSyncNotified()
+        {
+            return localSyncNotified;
         }
     }
 
-    @Override
-    public void acknowledgeEpoch(EpochReady ready)
+    static class EpochHistory extends AbstractConfigurationService.AbstractEpochHistory<EpochState>
     {
-        Topology acknowledged = getTopologyForEpoch(ready.epoch);
-        for (Node.Id node : acknowledged.nodes())
+        @Override
+        protected EpochState createEpochState(long epoch)
         {
-            if (node.equals(localId))
-                continue;
-
-            ready.coordination.addCallback(() -> {
-                for (Listener listener : listeners)
-                    listener.onEpochSyncComplete(node, ready.epoch);
-            });
+            return new EpochState(epoch);
         }
     }
 
-    public synchronized void createEpochFromConfig()
+    public AccordConfigurationService(Node.Id node, MessageDelivery messagingService, IFailureDetector failureDetector)
     {
-        Topology current = currentTopology();
-        Topology topology = AccordTopologyUtils.createTopology(epochs.size());
-        if (current.equals(topology.withEpoch(current.epoch()))) return;
-        unsafeAddEpoch(topology);
+        super(node);
+        this.messagingService = messagingService;
+        this.failureDetector = failureDetector;
     }
 
-    private void unsafeAddEpoch(Topology topology)
+    public AccordConfigurationService(Node.Id node)
     {
-        epochs.add(topology);
-        for (Listener listener : listeners)
-            listener.onTopologyUpdate(topology);
+        this(node, MessagingService.instance(), FailureDetector.instance);
+    }
 
-        // TODO: This is a hack to enable simplistic cluster reuse for TxnAuthTest, AccordCQLTest, etc.
-        // Since we don't have a dist sys that sets this up, we have to just lie...
-        EndpointMapping.knownIds().forEach(id -> {
-            for (Listener listener : listeners)
-                listener.onEpochSyncComplete(id, topology.epoch());
+    @Override
+    protected EpochHistory createEpochHistory()
+    {
+        return new EpochHistory();
+    }
+
+    public synchronized void start()
+    {
+        Invariants.checkState(state == State.INITIALIZED, "Expected state to be INITIALIZED but was %s", state);
+        state = State.LOADING;
+        updateMapping(ClusterMetadata.current());
+        diskState = AccordKeyspace.loadTopologies(((epoch, topology, syncStatus, pendingSyncNotify, remoteSyncComplete) -> {
+            if (topology != null)
+                reportTopology(topology, syncStatus == SyncStatus.NOT_STARTED);
+
+            getOrCreateEpochState(epoch).setSyncStatus(syncStatus);
+            if (syncStatus == SyncStatus.NOTIFYING)
+                syncNotifiers.put(epoch, new AccordLocalSyncNotifier(epoch, localId, pendingSyncNotify, this, messagingService, failureDetector, this));
+
+            remoteSyncComplete.forEach(id -> remoteSyncComplete(id, epoch));
+        }));
+        syncNotifiers.values().forEach(AccordLocalSyncNotifier::start);
+        state = State.STARTED;
+    }
+
+    @Override
+    public Node.Id mappedId(InetAddressAndPort endpoint)
+    {
+        return Invariants.nonNull(mapping.mappedId(endpoint));
+    }
+
+    @Override
+    public InetAddressAndPort mappedEndpoint(Node.Id id)
+    {
+        return Invariants.nonNull(mapping.mappedEndpoint(id));
+    }
+
+    @VisibleForTesting
+    EpochDiskState diskState()
+    {
+        return diskState;
+    }
+
+    @VisibleForTesting
+    synchronized void updateMapping(EndpointMapping mapping)
+    {
+        if (mapping.epoch() > this.mapping.epoch())
+            this.mapping = mapping;
+    }
+
+    synchronized void updateMapping(ClusterMetadata metadata)
+    {
+        updateMapping(AccordTopologyUtils.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory));
+    }
+
+    private void reportMetadata(ClusterMetadata metadata)
+    {
+        Stage.MISC.submit(() -> {
+            synchronized (AccordConfigurationService.this)
+            {
+                logger.info("Reporting metadata for epoch {}", metadata.epoch.getEpoch());
+                updateMapping(metadata);
+                reportTopology(AccordTopologyUtils.createAccordTopology(metadata, this::isAccordManagedKeyspace));
+            }
         });
     }
+
+    private void maybeReportMetadata(ClusterMetadata metadata)
+    {
+        // don't report metadata until the previous one has been acknowledged
+        synchronized (this)
+        {
+            long epoch = metadata.epoch.getEpoch();
+            if (epochs.maxEpoch() == 0)
+            {
+                getOrCreateEpochState(epoch);  // touch epoch state so subsequent calls see it
+                reportMetadata(metadata);
+                return;
+            }
+            getOrCreateEpochState(epoch - 1).acknowledged().addCallback(() -> reportMetadata(metadata));
+        }
+    }
+
+    @Override
+    public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
+    {
+        maybeReportMetadata(prev);
+        maybeReportMetadata(next);
+    }
+
+    @Override
+    protected void fetchTopologyInternal(long epoch)
+    {
+        // TODO: need a non-blocking way to inform CMS of an unknown epoch
+//        ClusterMetadataService.instance().maybeCatchup(Epoch.create(epoch));
+    }
+
+    @Override
+    protected synchronized void localSyncComplete(Topology topology)
+    {
+        long epoch = topology.epoch();
+        EpochState epochState = getOrCreateEpochState(epoch);
+        if (epochState.syncStatus != SyncStatus.NOT_STARTED)
+            return;
+
+        Set<Node.Id> pendingNotification = topology.nodes().stream().filter(i -> !localId.equals(i)).collect(Collectors.toSet());
+        AccordLocalSyncNotifier notifier = new AccordLocalSyncNotifier(epoch, localId, pendingNotification, this, messagingService, failureDetector, this);
+        syncNotifiers.put(epoch, notifier);
+        diskState = AccordKeyspace.setNotifyingLocalSync(epoch, pendingNotification, diskState);
+        epochState.setSyncStatus(SyncStatus.NOTIFYING);
+        notifier.start();
+    }
+
+    @Override
+    public long currentEpoch()
+    {
+        return super.currentEpoch();
+    }
+
+    @Override
+    public synchronized void onEndpointAck(Node.Id id, long epoch)
+    {
+        EpochState epochState = getOrCreateEpochState(epoch);
+        if (epochState.syncStatus != SyncStatus.NOTIFYING)
+            return;
+        diskState = AccordKeyspace.markLocalSyncAck(id, epoch, diskState);
+    }
+
+    @Override
+    public synchronized void onComplete(long epoch)
+    {
+        EpochState epochState = getOrCreateEpochState(epoch);
+        epochState.setSyncStatus(SyncStatus.COMPLETED);
+        diskState = AccordKeyspace.setCompletedLocalSync(epoch, diskState);
+    }
+
+    @Override
+    protected synchronized void topologyUpdatePreListenerNotify(Topology topology)
+    {
+        if (state == State.STARTED)
+            diskState = AccordKeyspace.saveTopology(topology, diskState);
+    }
+
+    @Override
+    protected void remoteSyncCompletePreListenerNotify(Node.Id node, long epoch)
+    {
+        if (state == State.STARTED)
+            diskState = AccordKeyspace.markRemoteTopologySync(node, epoch, diskState);
+    }
+
+    @Override
+    protected void truncateTopologiesPreListenerNotify(long epoch)
+    {
+        Invariants.checkState(state == State.STARTED);
+    }
+
+    @Override
+    protected void truncateTopologiesPostListenerNotify(long epoch)
+    {
+        if (state == State.STARTED)
+            diskState = AccordKeyspace.truncateTopologyUntil(epoch, diskState);
+    }
+
+    @VisibleForTesting
+    public static class EpochSnapshot
+    {
+        public enum ResultStatus
+        {
+            PENDING, SUCCESS, FAILURE;
+
+            static ResultStatus of (AsyncResult<?> result)
+            {
+                if (result == null || !result.isDone())
+                    return PENDING;
+
+                return result.isSuccess() ? SUCCESS : FAILURE;
+            }
+        }
+
+        public final long epoch;
+        public final SyncStatus syncStatus;
+        public final ResultStatus received;
+        public final ResultStatus acknowledged;
+        public final ResultStatus reads;
+
+        private EpochSnapshot(EpochState state)
+        {
+            this.epoch = state.epoch();
+            this.syncStatus = state.syncStatus;
+            this.received = ResultStatus.of(state.received());
+            this.acknowledged = ResultStatus.of(state.acknowledged());
+            this.reads = ResultStatus.of(state.reads());
+        }
+
+        public EpochSnapshot(long epoch, SyncStatus syncStatus, ResultStatus received, ResultStatus acknowledged, ResultStatus reads)
+        {
+            this.epoch = epoch;
+            this.syncStatus = syncStatus;
+            this.received = received;
+            this.acknowledged = acknowledged;
+            this.reads = reads;
+        }
+
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            EpochSnapshot that = (EpochSnapshot) o;
+            return epoch == that.epoch && syncStatus == that.syncStatus && received == that.received && acknowledged == that.acknowledged && reads == that.reads;
+        }
+
+        public int hashCode()
+        {
+            return Objects.hash(epoch, syncStatus, received, acknowledged, reads);
+        }
+
+        public String toString()
+        {
+            return "EpochSnapshot{" +
+                   "epoch=" + epoch +
+                   ", syncStatus=" + syncStatus +
+                   ", received=" + received +
+                   ", acknowledged=" + acknowledged +
+                   ", reads=" + reads +
+                   '}';
+        }
+
+        public static EpochSnapshot completed(long epoch)
+        {
+            return new EpochSnapshot(epoch, SyncStatus.COMPLETED, ResultStatus.SUCCESS, ResultStatus.SUCCESS, ResultStatus.SUCCESS);
+        }
+
+        public static EpochSnapshot notStarted(long epoch)
+        {
+            return new EpochSnapshot(epoch, SyncStatus.NOT_STARTED, ResultStatus.SUCCESS, ResultStatus.SUCCESS, ResultStatus.SUCCESS);
+        }
+    }
+
+    @VisibleForTesting
+    public synchronized EpochSnapshot getEpochSnapshot(long epoch)
+    {
+        if (epoch < epochs.minEpoch() || epoch > epochs.maxEpoch())
+            return null;
+
+        return new EpochSnapshot(getOrCreateEpochState(epoch));
+    }
+
+    @VisibleForTesting
+    public synchronized long minEpoch()
+    {
+        return epochs.minEpoch();
+    }
+
+    @VisibleForTesting
+    public synchronized long maxEpoch()
+    {
+        return epochs.maxEpoch();
+    }
+
+    @VisibleForTesting
+    public synchronized Future<Void> localSyncNotified(long epoch)
+    {
+        AsyncPromise<Void> promise = new AsyncPromise<>();
+        getOrCreateEpochState(epoch).localSyncNotified().addCallback((result, failure) -> {
+            if (failure != null) promise.tryFailure(failure);
+            else promise.trySuccess(result);
+        });
+        return promise;
+    }
+
+    public boolean isAccordManagedKeyspace(String keyspace)
+    {
+        // TODO (required, interop) : replace with schema flag or other mechanism for classifying accord keyspaces
+        return !SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspace);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordDataStore.java

@@ -23,25 +23,14 @@ import accord.local.Node;
 import accord.local.SafeCommandStore;
 import accord.primitives.Ranges;
 import accord.primitives.SyncPoint;
-import accord.primitives.Timestamp;
-import accord.utils.async.AsyncResults;
 
-public enum AccordDataStore implements DataStore
+public class AccordDataStore implements DataStore
 {
-    INSTANCE;
-
     @Override
     public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback)
     {
-        //TODO (implement): do real work
-        callback.starting(ranges).started(Timestamp.NONE);
-        callback.fetched(ranges);
-        return new ImmediateFetchFuture(ranges);
-    }
-
-    private static class ImmediateFetchFuture extends AsyncResults.SettableResult<Ranges> implements FetchResult
-    {
-        ImmediateFetchFuture(Ranges ranges) { setSuccess(ranges); }
-        @Override public void abort(Ranges abort) { }
+        AccordFetchCoordinator coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore());
+        coordinator.start();
+        return coordinator.result();
     }
 }

src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java

@@ -16,18 +16,16 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.tools.nodetool;
+package org.apache.cassandra.service.accord;
 
-import io.airlift.airline.Command;
-import org.apache.cassandra.tools.NodeProbe;
-import org.apache.cassandra.tools.NodeTool;
+import accord.local.Node;
+import org.apache.cassandra.locator.InetAddressAndPort;
 
-@Command(name="createepochunsafe", description = "manually create an Accord epoch from current topology")
-public class CreateEpochUnsafe extends NodeTool.NodeToolCmd
+/**
+ * Maps network addresses to accord ids
+ */
+public interface AccordEndpointMapper
 {
-    @Override
-    protected void execute(NodeProbe probe)
-    {
-        throw new UnsupportedOperationException("git rebase to pick up TCM removes this");
-    }
+    Node.Id mappedId(InetAddressAndPort endpoint);
+    InetAddressAndPort mappedEndpoint(Node.Id id);
 }

src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java

@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
+import accord.api.Data;
+import accord.api.DataStore;
+import accord.api.Query;
+import accord.api.Read;
+import accord.api.Update;
+import accord.impl.AbstractFetchCoordinator;
+import accord.local.CommandStore;
+import accord.local.Node;
+import accord.local.SafeCommandStore;
+import accord.primitives.PartialTxn;
+import accord.primitives.Range;
+import accord.primitives.Ranges;
+import accord.primitives.Routable;
+import accord.primitives.Seekable;
+import accord.primitives.Seekables;
+import accord.primitives.SyncPoint;
+import accord.primitives.Timestamp;
+import accord.primitives.Txn;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncChains;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.RangesAtEndpoint;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.service.accord.serializers.CommandSerializers;
+import org.apache.cassandra.service.accord.serializers.KeySerializers;
+import org.apache.cassandra.streaming.PreviewKind;
+import org.apache.cassandra.streaming.StreamCoordinator;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamOperation;
+import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.CastingSerializer;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap;
+import static org.apache.cassandra.utils.CollectionSerializers.serializeMap;
+import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize;
+
+public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamManager.StreamListener
+{
+    private static final Query noopQuery = (txnId, executeAt, data, read, update) -> null;
+
+    public static class StreamData implements Data
+    {
+        public static class SessionInfo
+        {
+            final TimeUUID planId;
+            final boolean hasData;
+
+            public SessionInfo(TimeUUID planId, boolean hasData)
+            {
+                this.planId = planId;
+                this.hasData = hasData;
+            }
+
+            static final IVersionedSerializer<SessionInfo> serializer = new IVersionedSerializer<SessionInfo>()
+            {
+                public void serialize(SessionInfo info, DataOutputPlus out, int version) throws IOException
+                {
+                    TimeUUID.Serializer.instance.serialize(info.planId, out, version);
+                    out.writeBoolean(info.hasData);
+
+                }
+
+                public SessionInfo deserialize(DataInputPlus in, int version) throws IOException
+                {
+                    return new SessionInfo(TimeUUID.Serializer.instance.deserialize(in, version), in.readBoolean());
+                }
+
+                public long serializedSize(SessionInfo info, int version)
+                {
+                    return TimeUUID.Serializer.instance.serializedSize(info.planId, version) + TypeSizes.BOOL_SIZE;
+                }
+            };
+        }
+        public static final IVersionedSerializer<StreamData> serializer = new IVersionedSerializer<StreamData>()
+        {
+            @Override
+            public void serialize(StreamData data, DataOutputPlus out, int version) throws IOException
+            {
+                serializeMap(data.streams, out, version, TokenRange.serializer, SessionInfo.serializer);
+            }
+
+            @Override
+            public StreamData deserialize(DataInputPlus in, int version) throws IOException
+            {
+
+                return new StreamData(ImmutableMap.copyOf(deserializeMap(in, version,
+                                                                         TokenRange.serializer,
+                                                                         SessionInfo.serializer)));
+            }
+
+            @Override
+            public long serializedSize(StreamData data, int version)
+            {
+                return serializedMapSize(data.streams, version, TokenRange.serializer, SessionInfo.serializer);
+            }
+        };
+
+        private final ImmutableMap<TokenRange, SessionInfo> streams;
+
+        public StreamData(ImmutableMap<TokenRange, SessionInfo> streams)
+        {
+            this.streams = streams;
+        }
+
+        public static StreamData of(TokenRange range, TimeUUID streamId, boolean hasData)
+        {
+            return new StreamData(ImmutableMap.of(range, new SessionInfo(streamId, hasData)));
+        }
+
+        @Override
+        public Data merge(Data data)
+        {
+            StreamData that = (StreamData) data;
+            if (that.streams.keySet().stream().anyMatch(this.streams::containsKey))
+                throw new IllegalStateException(String.format("Unable to merge: key found in multiple StreamData %s %s",
+                                                              this.streams.keySet(), that.streams.keySet()));
+            Invariants.checkState(!that.streams.keySet().stream().anyMatch(this.streams::containsKey));
+            ImmutableMap.Builder<TokenRange, SessionInfo> builder = ImmutableMap.builder();
+            builder.putAll(this.streams);
+            builder.putAll(that.streams);
+            return new StreamData(builder.build());
+        }
+    }
+
+    // needs to be externally synchronized
+    private class IncomingStream
+    {
+        private final TimeUUID planId;
+        private Range range;
+        private Node.Id from;
+        private StreamResultFuture future;
+
+        public IncomingStream(TimeUUID planId)
+        {
+            this.planId = planId;
+        }
+
+        private void rangeReceived(Range range, Node.Id from)
+        {
+            Invariants.nonNull(range);
+            Invariants.nonNull(from);
+            Invariants.checkState(this.range == null, "range was not null: %s", this.range);
+            Invariants.checkState(this.from == null, "from was not null: %s", this.from);
+            this.range = range;
+            this.from = from;
+            maybeListen();
+        }
+
+        private void futureReceived(StreamResultFuture future)
+        {
+            Invariants.nonNull(future);
+            Invariants.checkState(this.future == null, "future was not null: %s", this.future);
+            this.future = future;
+            maybeListen();
+        }
+
+        private void maybeListen()
+        {
+            if (range == null || future == null)
+                return;
+
+            Invariants.nonNull(from);
+
+            future.addCallback((state, fail) -> {
+                if (fail == null) success(from, Ranges.of(range));
+                else fail(from, Ranges.of(range), fail);
+            }, ((AccordCommandStore) commandStore()).executor());
+        }
+    }
+
+    public static class StreamingRead implements Read
+    {
+        public static final IVersionedSerializer<StreamingRead> serializer = new IVersionedSerializer<StreamingRead>()
+        {
+            @Override
+            public void serialize(StreamingRead read, DataOutputPlus out, int version) throws IOException
+            {
+                InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(read.to, out, version);
+                KeySerializers.ranges.serialize(read.ranges, out, version);
+            }
+
+            @Override
+            public StreamingRead deserialize(DataInputPlus in, int version) throws IOException
+            {
+                return new StreamingRead(InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version),
+                                         KeySerializers.ranges.deserialize(in, version));
+            }
+
+            @Override
+            public long serializedSize(StreamingRead read, int version)
+            {
+                return InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(read.to, version)
+                       + KeySerializers.ranges.serializedSize(read.ranges, version);
+            }
+        };
+
+        private final InetAddressAndPort to;
+        private final Ranges ranges;
+
+        public StreamingRead(InetAddressAndPort to, Ranges ranges)
+        {
+            this.to = to;
+            this.ranges = ranges;
+        }
+
+        @Override
+        public Seekables<?, ?> keys() { return ranges; }
+
+        private static boolean hasDataToStream(StreamCoordinator coordinator, InetAddressAndPort to)
+        {
+            for (StreamSession session : coordinator.getAllStreamSessions())
+            {
+                if (!session.peer.equals(to))
+                    continue;
+
+                Invariants.checkState(session.getNumRequests() == 0, "Requested to send data: %s", session);
+                if (session.getNumTransfers() > 0)
+                    return true;
+            }
+            return false;
+        }
+
+        @Override
+        public AsyncChain<Data> read(Seekable key, Txn.Kind kind, SafeCommandStore commandStore, Timestamp executeAt, DataStore store)
+        {
+            try
+            {
+                Invariants.checkArgument(key.domain() == Routable.Domain.Range, "Required Range but saw %s: %s", key.domain(), key);
+                TokenRange range = (TokenRange) key;
+
+                // TODO (correctness): check epoch
+                // TODO (correctness): handle dropped tables
+                KeyspaceMetadata ksm = ClusterMetadata.current().schema.getKeyspaceMetadata(range.keyspace());
+                Invariants.checkState(ksm != null, "Keyspace %s not found", range.keyspace());
+                Invariants.checkState(ksm.tables.size() > 0, "Keyspace '%s' has no tables", range.keyspace());
+
+                // FIXME: may also be relocation
+                StreamPlan plan = new StreamPlan(StreamOperation.BOOTSTRAP, 1, false,
+                                                 null, PreviewKind.NONE).flushBeforeTransfer(true);
+
+                RangesAtEndpoint ranges = RangesAtEndpoint.toDummyList(Collections.singleton(range.toKeyspaceRange()));
+                ksm.tables.forEach(table -> plan.transferRanges(to, table.keyspace, ranges, table.name));
+                StreamResultFuture future = plan.execute();
+                return AsyncChains.success(StreamData.of(range, future.planId, hasDataToStream(future.getCoordinator(), to)));
+            }
+            catch (Throwable t)
+            {
+                return AsyncChains.failure(t);
+            }
+        }
+
+        @Override
+        public Read slice(Ranges ranges) { return new StreamingRead(to, this.ranges.slice(ranges)); }
+
+        @Override
+        public Read merge(Read other) { throw new UnsupportedOperationException(); }
+    }
+
+    public static class StreamingTxn
+    {
+        private static final IVersionedSerializer<Read> read = new CastingSerializer<>(StreamingRead.class,
+                                                                                       StreamingRead.serializer);
+
+        private static final IVersionedSerializer<Query> query = new IVersionedSerializer<Query>()
+        {
+            @Override
+            public void serialize(Query t, DataOutputPlus out, int version)
+            {
+                Invariants.checkArgument(t == noopQuery);
+            }
+
+            @Override
+            public Query deserialize(DataInputPlus in, int version)
+            {
+                return noopQuery;
+            }
+
+            @Override
+            public long serializedSize(Query t, int version)
+            {
+                Invariants.checkArgument(t == noopQuery);
+                return 0;
+            }
+        };
+
+        private static final IVersionedSerializer<Update> update = new IVersionedSerializer<Update>()
+        {
+            @Override
+            public void serialize(Update t, DataOutputPlus out, int version)
+            {
+                Invariants.checkArgument(t == null);
+            }
+
+            @Override
+            public Update deserialize(DataInputPlus in, int version)
+            {
+                return null;
+            }
+
+            @Override
+            public long serializedSize(Update t, int version)
+            {
+                Invariants.checkArgument(t == null);
+                return 0;
+            }
+        };
+
+        // TODO (desired): this could be serialized as an InetAddressAndPort and Ranges if we had a special case PartialTxn implementation
+        public static final IVersionedSerializer<PartialTxn> serializer = new CommandSerializers.PartialTxnSerializer(read, query, update);
+    }
+
+    private final Map<TimeUUID, IncomingStream> streams = new HashMap<>();
+
+    public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore)
+    {
+        super(node, ranges, syncPoint, fetchRanges, commandStore);
+    }
+
+    @Override
+    public void start()
+    {
+        StreamManager.instance.addListener(this);
+        super.start();
+    }
+
+    private IncomingStream stream(TimeUUID id)
+    {
+        return streams.computeIfAbsent(id, IncomingStream::new);
+    }
+
+    // called from stream thread
+    @Override
+    public synchronized void onRegister(StreamResultFuture result)
+    {
+        stream(result.planId).futureReceived(result);
+    }
+
+    protected void onDone(Ranges success, Throwable failure)
+    {
+        StreamManager.instance.removeListener(this);
+        super.onDone(success, failure);
+    }
+
+    @Override
+    protected PartialTxn rangeReadTxn(Ranges ranges)
+    {
+        StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges);
+        return new PartialTxn.InMemory(ranges, Txn.Kind.Read, ranges, read, noopQuery, null);
+    }
+
+    @Override
+    protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges received)
+    {
+        if (data == null)
+            return;
+
+        StreamData streamData = (StreamData) data;
+        streamData.streams.forEach((range, streamInfo) -> {
+            if (streamInfo.hasData)
+            {
+                stream(streamInfo.planId).rangeReceived(range, from);
+            }
+            else
+            {
+                // if there was no data to stream, no connection is initiated, and we aren't notified via the stream
+                // listener, so the stream initiator notifies us and we mark it complete here
+                success(from, Ranges.of(range));
+            }
+        });
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordKeyspace.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.service.accord;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -31,13 +32,20 @@ import java.util.NavigableMap;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.slf4j.Logger;
@@ -56,24 +64,33 @@ import accord.local.Status;
 import accord.primitives.Ballot;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialTxn;
+import accord.primitives.Ranges;
 import accord.primitives.Routable;
 import accord.primitives.Route;
 import accord.primitives.Timestamp;
 import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.primitives.Writes;
+import accord.topology.Topology;
 import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
 import accord.utils.async.Observable;
 import org.apache.cassandra.concurrent.DebuggableTask;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
 import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
 import org.apache.cassandra.db.Clustering;
 import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Columns;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.IMutation;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.RegularAndStaticColumns;
@@ -119,19 +136,25 @@ import org.apache.cassandra.schema.Types;
 import org.apache.cassandra.schema.UserFunctions;
 import org.apache.cassandra.schema.Views;
 import org.apache.cassandra.serializers.UUIDSerializer;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey;
 import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.serializers.CommandSerializers;
+import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers;
 import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 import org.apache.cassandra.service.accord.serializers.DepsSerializer;
 import org.apache.cassandra.service.accord.serializers.KeySerializers;
 import org.apache.cassandra.service.accord.serializers.ListenerSerializers;
+import org.apache.cassandra.service.accord.serializers.TopologySerializers;
 import org.apache.cassandra.service.accord.txn.TxnData;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 import static java.lang.String.format;
 import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
 import static org.apache.cassandra.db.rows.BufferCell.live;
 import static org.apache.cassandra.db.rows.BufferCell.tombstone;
 import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
@@ -144,6 +167,9 @@ public class AccordKeyspace
 
     public static final String COMMANDS = "commands";
     public static final String COMMANDS_FOR_KEY = "commands_for_key";
+    public static final String TOPOLOGIES = "topologies";
+    public static final String EPOCH_METADATA = "epoch_metadata";
+    public static final String COMMAND_STORE_METADATA = "command_store_metadata";
 
     private static final TupleType TIMESTAMP_TYPE = new TupleType(Lists.newArrayList(LongType.instance, LongType.instance, Int32Type.instance));
     private static final String TIMESTAMP_TUPLE = TIMESTAMP_TYPE.asCQL3Type().toString();
@@ -226,7 +252,7 @@ public class AccordKeyspace
         .build();
 
     // TODO: naming is not very clearly distinct from the base serializers
-    private static class CommandsSerializers
+    private static class LocalVersionedSerializers
     {
         static final LocalVersionedSerializer<Route<?>> route = localSerializer(KeySerializers.route);
         static final LocalVersionedSerializer<AccordRoutingKey> routingKey = localSerializer(AccordRoutingKey.serializer);
@@ -235,6 +261,10 @@ public class AccordKeyspace
         static final LocalVersionedSerializer<Writes> writes = localSerializer(CommandSerializers.writes);
         static final LocalVersionedSerializer<TxnData> result = localSerializer(TxnData.serializer);
         static final LocalVersionedSerializer<Command.DurableAndIdempotentListener> listeners = localSerializer(ListenerSerializers.listener);
+        static final LocalVersionedSerializer<Topology> topology = localSerializer(TopologySerializers.topology);
+        static final LocalVersionedSerializer<ReducingRangeMap<Timestamp>> rejectBefore = localSerializer(CommandStoreSerializers.rejectBefore);
+        static final LocalVersionedSerializer<NavigableMap<TxnId, Ranges>> bootstrapBeganAt = localSerializer(CommandStoreSerializers.bootstrapBeganAt);
+        static final LocalVersionedSerializer<NavigableMap<Timestamp, Ranges>> safeToRead = localSerializer(CommandStoreSerializers.safeToRead);
 
         private static <T> LocalVersionedSerializer<T> localSerializer(IVersionedSerializer<T> serializer)
         {
@@ -246,7 +276,7 @@ public class AccordKeyspace
     {
         ColumnMetadata column = metadata.getColumn(new ColumnIdentifier(name, true));
         if (column == null)
-            throw new IllegalArgumentException(String.format("Unknown column %s for %s.%s", name, metadata.keyspace, metadata.name));
+            throw new IllegalArgumentException(format("Unknown column %s for %s.%s", name, metadata.keyspace, metadata.name));
         return column;
     }
 
@@ -339,6 +369,39 @@ public class AccordKeyspace
         }
     }
 
+    private static final TableMetadata Topologies =
+        parse(TOPOLOGIES,
+              "accord topologies",
+              "CREATE TABLE %s (" +
+              "epoch bigint primary key, " +
+              "topology blob, " +
+              "sync_state int, " +
+              "pending_sync_notify set<int>, " + // nodes that need to be told we're synced
+              "remote_sync_complete set<int> " +  // nodes that have told us they're synced
+              ')').build();
+
+    private static final TableMetadata EpochMetadata =
+        parse(EPOCH_METADATA,
+              "global epoch info",
+              "CREATE TABLE %s (" +
+              "key int primary key, " +
+              "min_epoch bigint, " +
+              "max_epoch bigint " +
+              ')').build();
+
+    private static final TableMetadata CommandStoreMetadata =
+        parse(COMMAND_STORE_METADATA,
+              "command store state",
+              "CREATE TABLE %s (" +
+              "store_id int, " +
+              "reject_before blob, " +
+              "bootstrap_began_at blob, " +
+              "safe_to_read blob, " +
+              "PRIMARY KEY(store_id)" +
+              ')').build();
+
+    private static final AtomicLong commandStoreMetadataTimestamp = new AtomicLong();
+
     private static TableMetadata.Builder parse(String name, String description, String cql)
     {
         return CreateTableStatement.parse(format(cql, name), ACCORD_KEYSPACE_NAME)
@@ -347,6 +410,11 @@ public class AccordKeyspace
                                    .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90));
     }
 
+    private static void flush(TableMetadata table)
+    {
+        Keyspace.open(table.keyspace).getColumnFamilyStore(table.id).forceBlockingFlush(ColumnFamilyStore.FlushReason.ACCORD);
+    }
+
     public static KeyspaceMetadata metadata()
     {
         return KeyspaceMetadata.create(ACCORD_KEYSPACE_NAME, KeyspaceParams.local(), tables(), Views.none(), Types.none(), UserFunctions.none());
@@ -354,7 +422,7 @@ public class AccordKeyspace
 
     private static Tables tables()
     {
-        return Tables.of(Commands, CommandsForKeys);
+        return Tables.of(Commands, CommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata);
     }
 
     private static <T> ByteBuffer serialize(T obj, LocalVersionedSerializer<T> serializer) throws IOException
@@ -364,7 +432,7 @@ public class AccordKeyspace
         {
             serializer.serialize(obj, out);
             ByteBuffer bb = out.buffer();
-            assert size == bb.limit() : String.format("Expected to write %d but wrote %d", size, bb.limit());
+            assert size == bb.limit() : format("Expected to write %d but wrote %d", size, bb.limit());
             return bb;
         }
     }
@@ -422,7 +490,7 @@ public class AccordKeyspace
         Listeners result = new Listeners();
         for (ByteBuffer bytes : serialized)
         {
-            result.add(deserialize(bytes, CommandsSerializers.listeners));
+            result.add(deserialize(bytes, LocalVersionedSerializers.listeners));
         }
         return new Listeners.Immutable(result);
     }
@@ -458,7 +526,7 @@ public class AccordKeyspace
 
     private static <C extends Command, V> void addKeyCellIfModified(ColumnMetadata column, Function<C, V> get, Row.Builder builder, long timestampMicros, C original, C command) throws IOException
     {
-        addCellIfModified(column, get, v -> serializeOrNull((AccordRoutingKey) v, CommandsSerializers.routingKey), builder, timestampMicros, original, command);
+        addCellIfModified(column, get, v -> serializeOrNull((AccordRoutingKey) v, LocalVersionedSerializers.routingKey), builder, timestampMicros, original, command);
     }
 
     private static <C extends Command, V extends Enum<V>> void addEnumCellIfModified(ColumnMetadata column, Function<C, V> get, Row.Builder builder, long timestampMicros, C original, C command) throws IOException
@@ -545,17 +613,17 @@ public class AccordKeyspace
             addEnumCellIfModified(CommandsColumns.status, Command::saveStatus, builder, timestampMicros, original, command);
             addKeyCellIfModified(CommandsColumns.home_key, Command::homeKey, builder, timestampMicros, original, command);
             addKeyCellIfModified(CommandsColumns.progress_key, Command::progressKey, builder, timestampMicros, original, command);
-            addCellIfModified(CommandsColumns.route, Command::route, CommandsSerializers.route, builder, timestampMicros, original, command);
+            addCellIfModified(CommandsColumns.route, Command::route, LocalVersionedSerializers.route, builder, timestampMicros, original, command);
             addEnumCellIfModified(CommandsColumns.durability, Command::durability, builder, timestampMicros, original, command);
-            addCellIfModified(CommandsColumns.txn, Command::partialTxn, CommandsSerializers.partialTxn, builder, timestampMicros, original, command);
+            addCellIfModified(CommandsColumns.txn, Command::partialTxn, LocalVersionedSerializers.partialTxn, builder, timestampMicros, original, command);
 
             addCellIfModified(CommandsColumns.execute_at, Command::executeAt, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command);
             addCellIfModified(CommandsColumns.promised_ballot, Command::promised, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command);
             addCellIfModified(CommandsColumns.accepted_ballot, Command::accepted, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command);
 
-            addCellIfModified(CommandsColumns.dependencies, Command::partialDeps, CommandsSerializers.partialDeps, builder, timestampMicros, original, command);
+            addCellIfModified(CommandsColumns.dependencies, Command::partialDeps, LocalVersionedSerializers.partialDeps, builder, timestampMicros, original, command);
 
-            addSetChanges(CommandsColumns.listeners, Command::durableListeners, v -> serialize(v, CommandsSerializers.listeners), builder, timestampMicros, nowInSeconds, original, command);
+            addSetChanges(CommandsColumns.listeners, Command::durableListeners, v -> serialize(v, LocalVersionedSerializers.listeners), builder, timestampMicros, nowInSeconds, original, command);
 
             if (command.isCommitted())
             {
@@ -567,8 +635,8 @@ public class AccordKeyspace
                 {
                     Command.Executed executed = command.asExecuted();
                     Command.Executed originalExecuted = original != null && original.isExecuted() ? original.asExecuted() : null;
-                    addCellIfModified(CommandsColumns.writes, Command.Executed::writes, v -> serialize(v, CommandsSerializers.writes), builder, timestampMicros, originalExecuted, executed);
-                    addCellIfModified(CommandsColumns.result, Command.Executed::result, v -> serialize((TxnData) v, CommandsSerializers.result), builder, timestampMicros, originalExecuted, executed);
+                    addCellIfModified(CommandsColumns.writes, Command.Executed::writes, v -> serialize(v, LocalVersionedSerializers.writes), builder, timestampMicros, originalExecuted, executed);
+                    addCellIfModified(CommandsColumns.result, Command.Executed::result, v -> serialize((TxnData) v, LocalVersionedSerializers.result), builder, timestampMicros, originalExecuted, executed);
                 }
             }
 
@@ -663,7 +731,7 @@ public class AccordKeyspace
                      "AND domain = ? " +
                      "AND txn_id=(?, ?, ?)";
 
-        return executeInternal(String.format(cql, ACCORD_KEYSPACE_NAME, COMMANDS),
+        return executeInternal(format(cql, ACCORD_KEYSPACE_NAME, COMMANDS),
                                commandStore.id(),
                                txnId.domain().ordinal(),
                                txnId.msb, txnId.lsb, txnId.node.id);
@@ -741,7 +809,7 @@ public class AccordKeyspace
         @Override
         public String description()
         {
-            return String.format("Table Walker for %s; queries = %d", getClass().getSimpleName(), numQueries);
+            return format("Table Walker for %s; queries = %d", getClass().getSimpleName(), numQueries);
         }
     }
 
@@ -785,7 +853,7 @@ public class AccordKeyspace
             super(executor, callback);
             this.storeId = commandStore;
             this.domain = domain.ordinal();
-            cql = String.format("SELECT %s " +
+            cql = format("SELECT %s " +
                                 "FROM %s " +
                                 "WHERE store_id = ? " +
                                 "      AND domain = ? " +
@@ -839,14 +907,14 @@ public class AccordKeyspace
             this.end = end;
 
             String selection = selection(CommandsForKeys, requiredColumns, COLUMNS_FOR_ITERATION);
-            this.cqlFirst = String.format("SELECT DISTINCT %s\n" +
+            this.cqlFirst = format("SELECT DISTINCT %s\n" +
                                           "FROM %s\n" +
                                           "WHERE store_id = ?\n" +
                                           (startInclusive ? "  AND key_token >= ?\n" : "  AND key_token > ?\n") +
                                           (endInclusive ? "  AND key_token <= ?\n" : "  AND key_token < ?\n") +
                                           "ALLOW FILTERING",
                                           selection, CommandsForKeys);
-            this.cqlContinue = String.format("SELECT DISTINCT %s\n" +
+            this.cqlContinue = format("SELECT DISTINCT %s\n" +
                                              "FROM %s\n" +
                                              "WHERE store_id = ?\n" +
                                              "  AND key_token > ?\n" +
@@ -891,9 +959,9 @@ public class AccordKeyspace
             CommonAttributes.Mutable attributes = new CommonAttributes.Mutable(txnId);
             // TODO: something less brittle than ordinal, more efficient than values()
             attributes.durability(Status.Durability.values()[row.getInt("durability", 0)]);
-            attributes.homeKey(deserializeOrNull(row.getBlob("home_key"), CommandsSerializers.routingKey));
-            attributes.progressKey(deserializeOrNull(row.getBlob("progress_key"), CommandsSerializers.routingKey));
-            attributes.route(deserializeOrNull(row.getBlob("route"), CommandsSerializers.route));
+            attributes.homeKey(deserializeOrNull(row.getBlob("home_key"), LocalVersionedSerializers.routingKey));
+            attributes.progressKey(deserializeOrNull(row.getBlob("progress_key"), LocalVersionedSerializers.routingKey));
+            attributes.route(deserializeOrNull(row.getBlob("route"), LocalVersionedSerializers.route));
             attributes.partialTxn(deserializeTxn(row));
             attributes.partialDeps(deserializeDependencies(row));
             attributes.setListeners(deserializeListeners(row, "listeners"));
@@ -903,8 +971,8 @@ public class AccordKeyspace
             Ballot accepted = deserializeTimestampOrNull(row, "accepted_ballot", Ballot::fromBits);
             ImmutableSortedSet<TxnId> waitingOnCommit = deserializeTxnIdNavigableSet(row, "waiting_on_commit");
             ImmutableSortedMap<Timestamp, TxnId> waitingOnApply = deserializeWaitingOnApply(row.getMap("waiting_on_apply", BytesType.instance, BytesType.instance));
-            Writes writes = deserializeWithVersionOr(row, "writes", CommandsSerializers.writes, () -> null);
-            Result result = deserializeWithVersionOr(row, "result", CommandsSerializers.result, () -> null);
+            Writes writes = deserializeWithVersionOr(row, "writes", LocalVersionedSerializers.writes, () -> null);
+            Result result = deserializeWithVersionOr(row, "result", LocalVersionedSerializers.result, () -> null);
 
             switch (status.status)
             {
@@ -941,7 +1009,7 @@ public class AccordKeyspace
 
     public static PartialDeps deserializeDependencies(UntypedResultSet.Row row) throws IOException
     {
-        return deserializeOrNull(row.getBlob("dependencies"), CommandsSerializers.partialDeps);
+        return deserializeOrNull(row.getBlob("dependencies"), LocalVersionedSerializers.partialDeps);
     }
 
     public static Timestamp deserializeExecuteAt(UntypedResultSet.Row row)
@@ -961,7 +1029,7 @@ public class AccordKeyspace
 
     public static PartialTxn deserializeTxn(UntypedResultSet.Row row) throws IOException
     {
-        return deserializeOrNull(row.getBlob("txn"), CommandsSerializers.partialTxn);
+        return deserializeOrNull(row.getBlob("txn"), LocalVersionedSerializers.partialTxn);
     }
 
     public static PartitionKey deserializeKey(UntypedResultSet.Row row)
@@ -1094,7 +1162,7 @@ public class AccordKeyspace
     private static ByteBuffer cellValue(Row row, ColumnMetadata column)
     {
         Cell<?> cell = row.getCell(column);
-        return  (cell != null && !cell.isTombstone()) ? cellValue(cell) : null;
+        return (cell != null && !cell.isTombstone()) ? cellValue(cell) : null;
     }
 
     private static <T> ByteBuffer clusteringValue(Clustering<T> clustering, int idx)
@@ -1124,8 +1192,8 @@ public class AccordKeyspace
         for (SeriesKind kind : SeriesKind.values())
             seriesMaps.put(kind, new ImmutableSortedMap.Builder<>(Comparator.naturalOrder()));
 
-        try(ReadExecutionController controller = command.executionController();
-            FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
+        try (ReadExecutionController controller = command.executionController();
+             FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
         {
             if (!partitions.hasNext())
             {
@@ -1175,4 +1243,297 @@ public class AccordKeyspace
             throw t;
         }
     }
+
+    public static class EpochDiskState
+    {
+        public static final EpochDiskState EMPTY = new EpochDiskState(0, 0);
+        public final long minEpoch;
+        public final long maxEpoch;
+
+        public EpochDiskState(long minEpoch, long maxEpoch)
+        {
+            Invariants.checkArgument(minEpoch >= 0, "Min Epoch %d < 0", minEpoch);
+            Invariants.checkArgument(maxEpoch >= minEpoch, "Max epoch %d < min %d", maxEpoch, minEpoch);
+            this.minEpoch = minEpoch;
+            this.maxEpoch = maxEpoch;
+        }
+
+        private EpochDiskState withNewMaxEpoch(long epoch)
+        {
+            Invariants.checkArgument(epoch > maxEpoch, "Epoch %d <= %d (max)", epoch, maxEpoch);
+            return new EpochDiskState(Math.max(1, minEpoch), epoch);
+        }
+
+        private EpochDiskState withNewMinEpoch(long epoch)
+        {
+            Invariants.checkArgument(epoch > minEpoch, "epoch %d <= %d (min)", epoch, minEpoch);
+            Invariants.checkArgument(epoch <= maxEpoch, "epoch %d > %d (max)", epoch, maxEpoch);
+            return new EpochDiskState(epoch, maxEpoch);
+        }
+
+        @Override
+        public String toString()
+        {
+            return "EpochDiskState{" +
+                   "minEpoch=" + minEpoch +
+                   ", maxEpoch=" + maxEpoch +
+                   '}';
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            EpochDiskState diskState = (EpochDiskState) o;
+            return minEpoch == diskState.minEpoch && maxEpoch == diskState.maxEpoch;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    private static void saveEpochDiskState(EpochDiskState diskState)
+    {
+        String cql = "INSERT INTO %s.%s (key, min_epoch, max_epoch) VALUES (0, ?, ?);";
+        executeInternal(format(cql, ACCORD_KEYSPACE_NAME, EPOCH_METADATA),
+                        diskState.minEpoch, diskState.maxEpoch);
+    }
+
+    @Nullable
+    @VisibleForTesting
+    public static EpochDiskState loadEpochDiskState()
+    {
+        String cql = "SELECT * FROM %s.%s WHERE key=0";
+        UntypedResultSet result = executeInternal(format(cql, ACCORD_KEYSPACE_NAME, EPOCH_METADATA));
+        if (result.isEmpty())
+            return null;
+        UntypedResultSet.Row row = result.one();
+        return new EpochDiskState(row.getLong("min_epoch"), row.getLong("max_epoch"));
+    }
+
+    /**
+     * Update the disk state for this epoch, if it's higher than the one we have one disk.
+     *
+     * This is meant to be called before any update involving the new epoch, not after. This way if the update
+     * fails, we can detect and cleanup. If we updated disk state after an update and it failed, we could "forget"
+     * about (now acked) topology updates after a restart.
+     */
+    private static EpochDiskState maybeUpdateMaxEpoch(EpochDiskState diskState, long epoch)
+    {
+        Invariants.checkArgument(epoch >= diskState.minEpoch, "Epoch %d < %d (min)", epoch, diskState.minEpoch);
+        if (epoch > diskState.maxEpoch)
+        {
+            diskState = diskState.withNewMaxEpoch(epoch);
+            saveEpochDiskState(diskState);
+        }
+        return diskState;
+    }
+
+    public static EpochDiskState saveTopology(Topology topology, EpochDiskState diskState)
+    {
+        diskState = maybeUpdateMaxEpoch(diskState, topology.epoch());
+
+        try
+        {
+            String cql = "UPDATE %s.%s SET topology=? WHERE epoch=?";
+            executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES),
+                            serialize(topology, LocalVersionedSerializers.topology), topology.epoch());
+            flush(Topologies);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+
+        return diskState;
+    }
+
+    public static EpochDiskState markRemoteTopologySync(Node.Id node, long epoch, EpochDiskState diskState)
+    {
+        diskState = maybeUpdateMaxEpoch(diskState, epoch);
+        String cql = "UPDATE %s.%s SET remote_sync_complete = remote_sync_complete + ? WHERE epoch = ?";
+        executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES),
+                        Collections.singleton(node.id), epoch);
+        flush(Topologies);
+        return diskState;
+    }
+
+    public static EpochDiskState setNotifyingLocalSync(long epoch, Set<Node.Id> pending, EpochDiskState diskState)
+    {
+        diskState = maybeUpdateMaxEpoch(diskState, epoch);
+        String cql = "UPDATE %s.%s SET sync_state = ?, pending_sync_notify = ? WHERE epoch = ?";
+        executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES),
+                        SyncStatus.NOTIFYING.ordinal(),
+                        pending.stream().map(i -> i.id).collect(Collectors.toSet()),
+                        epoch);
+        return diskState;
+    }
+
+    public static EpochDiskState markLocalSyncAck(Node.Id node, long epoch, EpochDiskState diskState)
+    {
+        diskState = maybeUpdateMaxEpoch(diskState, epoch);
+        String cql = "UPDATE %s.%s SET pending_sync_notify = pending_sync_notify - ? WHERE epoch = ?";
+        executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES),
+                        Collections.singleton(node.id), epoch);
+        return diskState;
+    }
+
+    public static EpochDiskState setCompletedLocalSync(long epoch, EpochDiskState diskState)
+    {
+        diskState = maybeUpdateMaxEpoch(diskState, epoch);
+        String cql = "UPDATE %s.%s SET sync_state = ?, pending_sync_notify = {} WHERE epoch = ?";
+        executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES),
+                        SyncStatus.COMPLETED.ordinal(),
+                        epoch);
+        return diskState;
+    }
+
+    public static EpochDiskState truncateTopologyUntil(final long epoch, EpochDiskState diskState)
+    {
+        while (diskState.minEpoch < epoch)
+        {
+            long delete = diskState.minEpoch;
+            diskState = diskState.withNewMinEpoch(delete + 1);
+            saveEpochDiskState(diskState);
+            String cql = "DELETE FROM %s.%s WHERE epoch = ?";
+            executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), delete);
+        }
+        return diskState;
+    }
+
+    public interface TopologyLoadConsumer
+    {
+        void load(long epoch, Topology topology, SyncStatus syncStatus, Set<Node.Id> pendingSyncNotify, Set<Node.Id> remoteSyncComplete);
+    }
+
+    @VisibleForTesting
+    public static void loadEpoch(long epoch, TopologyLoadConsumer consumer) throws IOException
+    {
+        String cql = format("SELECT * FROM %s.%s WHERE epoch=?", ACCORD_KEYSPACE_NAME, TOPOLOGIES);
+
+        UntypedResultSet result = executeInternal(cql, epoch);
+        Invariants.checkState(!result.isEmpty(), "Nothing found for epoch %d", epoch);
+        UntypedResultSet.Row row = result.one();
+        Topology topology = row.has("topology")
+                            ? deserialize(row.getBytes("topology"), LocalVersionedSerializers.topology)
+                            : null;
+
+        SyncStatus syncStatus = row.has("sync_state")
+                                ? SyncStatus.values()[row.getInt("sync_state")]
+                                : SyncStatus.NOT_STARTED;
+        Set<Node.Id> pendingSyncNotify = row.has("pending_sync_notify")
+                                         ? row.getSet("pending_sync_notify", Int32Type.instance).stream().map(Node.Id::new).collect(Collectors.toSet())
+                                         : Collections.emptySet();
+        Set<Node.Id> remoteSyncComplete = row.has("remote_sync_complete")
+                                          ? row.getSet("remote_sync_complete", Int32Type.instance).stream().map(Node.Id::new).collect(Collectors.toSet())
+                                          : Collections.emptySet();
+
+        consumer.load(epoch, topology, syncStatus, pendingSyncNotify, remoteSyncComplete);
+
+    }
+
+    public static EpochDiskState loadTopologies(TopologyLoadConsumer consumer)
+    {
+        try
+        {
+            EpochDiskState diskState = loadEpochDiskState();
+            if (diskState == null)
+                return EpochDiskState.EMPTY;
+
+            for (long epoch=diskState.minEpoch; epoch<=diskState.maxEpoch; epoch++)
+                loadEpoch(epoch, consumer);
+
+            return diskState;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static IMutation getCommandStoreMetadataMutation(String cql, ByteBuffer... values)
+    {
+        ClientState clientState = ClientState.forInternalCalls();
+        ModificationStatement statement = (ModificationStatement) QueryProcessor.parseStatement(cql).prepare(ClientState.forInternalCalls());
+        QueryOptions options = QueryOptions.forInternalCalls(Arrays.asList(values));
+
+        long tsMicros = TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
+
+        while (true)
+        {
+            long prev = commandStoreMetadataTimestamp.get();
+            if (prev >= tsMicros)
+                tsMicros = prev + 1;
+
+            if (commandStoreMetadataTimestamp.compareAndSet(prev, tsMicros))
+                break;
+        }
+
+        return Iterables.getOnlyElement(statement.getMutations(clientState, options, true, tsMicros, (int) TimeUnit.MICROSECONDS.toSeconds(tsMicros), Dispatcher.RequestTime.forImmediateExecution()));
+    }
+
+
+    private static <T> Future<?> updateCommandStoreMetadata(CommandStore commandStore, String column, T value, LocalVersionedSerializer<T> serializer)
+    {
+        String cql = format("UPDATE %s.%s SET %s=? WHERE store_id=?", ACCORD_KEYSPACE_NAME, COMMAND_STORE_METADATA, column);
+        try
+        {
+            IMutation mutation = getCommandStoreMetadataMutation(cql, serialize(value, serializer), bytes(commandStore.id()));
+            return Stage.MUTATION.submit(mutation::apply);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static Future<?> updateRejectBefore(CommandStore commandStore, ReducingRangeMap<Timestamp> rejectBefore)
+    {
+        return updateCommandStoreMetadata(commandStore, "reject_before", rejectBefore, LocalVersionedSerializers.rejectBefore);
+    }
+
+    public static Future<?> updateBootstrapBeganAt(CommandStore commandStore, NavigableMap<TxnId, Ranges> bootstrapBeganAt)
+    {
+        return updateCommandStoreMetadata(commandStore, "bootstrap_began_at", bootstrapBeganAt, LocalVersionedSerializers.bootstrapBeganAt);
+    }
+
+    public static Future<?> updateSafeToRead(CommandStore commandStore, NavigableMap<Timestamp, Ranges> safeToRead)
+    {
+        return updateCommandStoreMetadata(commandStore, "safe_to_read", safeToRead, LocalVersionedSerializers.safeToRead);
+    }
+
+    public interface CommandStoreMetadataConsumer
+    {
+        void accept(ReducingRangeMap<Timestamp> rejectBefore, NavigableMap<TxnId, Ranges> bootstrapBeganAt, NavigableMap<Timestamp, Ranges> safeToRead);
+
+    }
+    public static void loadCommandStoreMetadata(int id, CommandStoreMetadataConsumer consumer)
+    {
+        UntypedResultSet result = executeOnceInternal(format("SELECT * FROM %s.%s WHERE store_id=?", ACCORD_KEYSPACE_NAME, COMMAND_STORE_METADATA), id);
+        ReducingRangeMap<Timestamp> rejectBefore = null;
+        NavigableMap<TxnId, Ranges> bootstrapBeganAt = null;
+        NavigableMap<Timestamp, Ranges> safeToRead = null;
+        if (!result.isEmpty())
+        {
+            UntypedResultSet.Row row = Iterables.getOnlyElement(result);
+            try
+            {
+                if (row.has("reject_before"))
+                    rejectBefore = deserialize(row.getBlob("reject_before"), LocalVersionedSerializers.rejectBefore);
+                if (row.has("bootstrap_began_at"))
+                    bootstrapBeganAt = deserialize(row.getBlob("bootstrap_began_at"), LocalVersionedSerializers.bootstrapBeganAt);
+                if (row.has("safe_to_read"))
+                    safeToRead = deserialize(row.getBlob("safe_to_read"), LocalVersionedSerializers.safeToRead);
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException(e);
+            }
+        }
+        consumer.accept(rejectBefore, bootstrapBeganAt, safeToRead);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordLocalSyncNotifier.java

@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import accord.local.Node;
+import accord.utils.Invariants;
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageDelivery;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.service.accord.serializers.TopologySerializers;
+
+public class AccordLocalSyncNotifier implements RequestCallback<AccordLocalSyncNotifier.Acknowledgement>
+{
+    public static final IVerbHandler<Notification> verbHandler = message -> AccordService.instance().remoteSyncComplete(message);
+    private static final Logger logger = LoggerFactory.getLogger(AccordLocalSyncNotifier.class);
+
+    interface Listener
+    {
+        void onEndpointAck(Node.Id id, long epoch);
+        void onComplete(long epoch);
+    }
+
+    private final long epoch;
+    private final Node.Id from;
+    private final Set<Node.Id> pendingNotifications;
+    private final AccordEndpointMapper endpointMapper;
+    private final IFailureDetector failureDetector;
+    private final Listener listener;
+    private final MessageDelivery messagingService;
+
+    public AccordLocalSyncNotifier(long epoch,
+                                   Node.Id from, Set<Node.Id> pendingNotifications,
+                                   AccordEndpointMapper endpointMapper,
+                                   MessageDelivery messagingService, IFailureDetector failureDetector,
+                                   Listener listener)
+    {
+        this.epoch = epoch;
+        this.from = from;
+        this.pendingNotifications = pendingNotifications;
+        this.endpointMapper = endpointMapper;
+        this.failureDetector = failureDetector;
+        this.listener = listener;
+        this.messagingService = messagingService;
+    }
+
+    private void notify(Node.Id to)
+    {
+        InetAddressAndPort toEp = endpointMapper.mappedEndpoint(to);
+        if (failureDetector.isAlive(toEp))
+        {
+            Message<Notification> msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, new Notification(epoch, from, to));
+            messagingService.sendWithCallback(msg, toEp, this);
+        }
+        else
+        {
+            scheduleNotify(to);
+        }
+    }
+
+    public void scheduleNotify(Node.Id to)
+    {
+        ScheduledExecutors.scheduledTasks.schedule(() -> notify(to), 1, TimeUnit.MINUTES);
+    }
+
+    public synchronized void start()
+    {
+        if (pendingNotifications.isEmpty())
+        {
+            listener.onComplete(epoch);
+            return;
+        }
+        pendingNotifications.forEach(this::notify);
+    }
+
+    private synchronized void onResponse(InetAddressAndPort fromEp, Node.Id from)
+    {
+        try
+        {
+            Invariants.checkArgument(endpointMapper.mappedId(fromEp).equals(from), "%s != %s", from, endpointMapper.mappedId(fromEp));
+            listener.onEndpointAck(from, epoch);
+            pendingNotifications.remove(from);
+            if (pendingNotifications.isEmpty())
+                listener.onComplete(epoch);
+        }
+        catch (Throwable t)
+        {
+            logger.error("Unhandled exception handling sync ack on epoch {} from {}", epoch, fromEp, t);
+            scheduleNotify(from);
+        }
+    }
+
+    @Override
+    public synchronized void onResponse(Message<Acknowledgement> msg)
+    {
+        onResponse(msg.from(), msg.payload.from);
+    }
+
+    @Override
+    public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
+    {
+        scheduleNotify(endpointMapper.mappedId(from));
+    }
+
+    public static class Notification
+    {
+        public static final IVersionedSerializer<Notification> serializer = new IVersionedSerializer<Notification>()
+        {
+            @Override
+            public void serialize(Notification notification, DataOutputPlus out, int version) throws IOException
+            {
+                out.writeLong(notification.epoch);
+                TopologySerializers.nodeId.serialize(notification.from, out, version);
+                TopologySerializers.nodeId.serialize(notification.to, out, version);
+            }
+
+            @Override
+            public Notification deserialize(DataInputPlus in, int version) throws IOException
+            {
+                return new Notification(in.readLong(),
+                                        TopologySerializers.nodeId.deserialize(in, version),
+                                        TopologySerializers.nodeId.deserialize(in, version));
+            }
+
+            @Override
+            public long serializedSize(Notification notification, int version)
+            {
+                return TypeSizes.LONG_SIZE
+                       + TopologySerializers.nodeId.serializedSize()
+                       + TopologySerializers.nodeId.serializedSize();
+            }
+        };
+        final long epoch;
+        final Node.Id from;
+        final Node.Id to;
+
+        public Notification(long epoch, Node.Id from, Node.Id to)
+        {
+            this.epoch = epoch;
+            this.from = from;
+            this.to = to;
+        }
+    }
+
+    public static class Acknowledgement
+    {
+        public static final IVersionedSerializer<Acknowledgement> serializer = new IVersionedSerializer<Acknowledgement>()
+        {
+            @Override
+            public void serialize(Acknowledgement acknowledgement, DataOutputPlus out, int version) throws IOException
+            {
+                TopologySerializers.nodeId.serialize(acknowledgement.from, out, version);
+            }
+
+            @Override
+            public Acknowledgement deserialize(DataInputPlus in, int version) throws IOException
+            {
+                return new Acknowledgement(TopologySerializers.nodeId.deserialize(in, version));
+            }
+
+            @Override
+            public long serializedSize(Acknowledgement acknowledgement, int version)
+            {
+                return TopologySerializers.nodeId.serializedSize();
+            }
+        };
+
+        final Node.Id from;
+
+        public Acknowledgement(Node.Id from)
+        {
+            this.from = from;
+        }
+    }
+}

src/java/org/apache/cassandra/service/accord/AccordMessageSink.java

@@ -38,8 +38,6 @@ import accord.messages.ReplyContext;
 import accord.messages.Request;
 import org.apache.cassandra.locator.InetAddressAndPort;
 
-import static org.apache.cassandra.service.accord.EndpointMapping.getEndpoint;
-
 public class AccordMessageSink implements MessageSink
 {
     private static final Logger logger = LoggerFactory.getLogger(AccordMessageSink.class);
@@ -77,6 +75,8 @@ public class AccordMessageSink implements MessageSink
             mapping.put(MessageType.GET_DEPS_REQ,           Verb.ACCORD_GET_DEPS_REQ);
             mapping.put(MessageType.GET_DEPS_RSP,           Verb.ACCORD_GET_DEPS_RSP);
             mapping.put(MessageType.SIMPLE_RSP,             Verb.ACCORD_SIMPLE_RSP);
+            mapping.put(MessageType.FETCH_DATA_REQ,         Verb.ACCORD_FETCH_DATA_REQ);
+            mapping.put(MessageType.FETCH_DATA_RSP,         Verb.ACCORD_FETCH_DATA_RSP);
 
             for (MessageType type : MessageType.values())
             {
@@ -93,16 +93,18 @@ public class AccordMessageSink implements MessageSink
 
     private final Agent agent;
     private final MessageDelivery messaging;
+    private final AccordEndpointMapper endpointMapper;
 
-    public AccordMessageSink(Agent agent, MessageDelivery messaging)
+    public AccordMessageSink(Agent agent, MessageDelivery messaging, AccordEndpointMapper endpointMapper)
     {
         this.agent = agent;
         this.messaging = messaging;
+        this.endpointMapper = endpointMapper;
     }
 
-    public AccordMessageSink(Agent agent)
+    public AccordMessageSink(Agent agent, AccordConfigurationService endpointMapper)
     {
-        this(agent, MessagingService.instance());
+        this(agent, MessagingService.instance(), endpointMapper);
     }
 
     @Override
@@ -111,7 +113,7 @@ public class AccordMessageSink implements MessageSink
         Verb verb = getVerb(request.type());
         Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
         Message<Request> message = Message.out(verb, request);
-        InetAddressAndPort endpoint = getEndpoint(to);
+        InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
         logger.debug("Sending {} {} to {}", verb, message.payload, endpoint);
         messaging.send(message, endpoint);
     }
@@ -122,9 +124,9 @@ public class AccordMessageSink implements MessageSink
         Verb verb = getVerb(request.type());
         Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
         Message<Request> message = Message.out(verb, request);
-        InetAddressAndPort endpoint = getEndpoint(to);
+        InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
         logger.debug("Sending {} {} to {}", verb, message.payload, endpoint);
-        messaging.sendWithCallback(message, endpoint, new AccordCallback<>(executor, (Callback<Reply>) callback));
+        messaging.sendWithCallback(message, endpoint, new AccordCallback<>(executor, (Callback<Reply>) callback, endpointMapper));
     }
 
     @Override
@@ -135,7 +137,7 @@ public class AccordMessageSink implements MessageSink
         Verb verb = getVerb(reply.type());
         Preconditions.checkNotNull(verb, "Verb is null for type %s", reply.type());
         Preconditions.checkArgument(replyMsg.verb() == verb, "Expected reply message with verb %s but got %s; reply type was %s", replyMsg.verb(), verb, reply.type());
-        InetAddressAndPort endpoint = getEndpoint(replyingToNode);
+        InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode);
         logger.debug("Replying {} {} to {}", replyMsg.verb(), replyMsg.payload, endpoint);
         messaging.send(replyMsg, endpoint);
     }

src/java/org/apache/cassandra/service/accord/AccordService.java

@@ -40,6 +40,8 @@ import accord.primitives.Txn;
 import accord.primitives.TxnId;
 import accord.topology.TopologyManager;
 import accord.utils.DefaultRandom;
+import accord.utils.Invariants;
+import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 import org.apache.cassandra.concurrent.Shutdownable;
 import accord.utils.async.AsyncResult;
@@ -50,15 +52,22 @@ import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.metrics.AccordClientRequestMetrics;
 import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.accord.api.AccordAgent;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter;
 import org.apache.cassandra.service.accord.api.AccordScheduler;
 import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
 import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
 import org.apache.cassandra.service.accord.txn.TxnData;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.ExecutorUtils;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.ImmediateFuture;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
 import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
@@ -70,12 +79,14 @@ public class AccordService implements IAccordService, Shutdownable
 
     public static final AccordClientRequestMetrics readMetrics = new AccordClientRequestMetrics("AccordRead");
     public static final AccordClientRequestMetrics writeMetrics = new AccordClientRequestMetrics("AccordWrite");
+    private static final Future<Void> BOOTSTRAP_SUCCESS = ImmediateFuture.success(null);
 
     private final Node node;
     private final Shutdownable nodeShutdown;
     private final AccordMessageSink messageSink;
     private final AccordConfigurationService configService;
     private final AccordScheduler scheduler;
+    private final AccordDataStore dataStore;
     private final AccordVerbHandler<? extends Request> verbHandler;
 
     private static final IAccordService NOOP_SERVICE = new IAccordService()
@@ -86,9 +97,6 @@ public class AccordService implements IAccordService, Shutdownable
             return null;
         }
 
-        @Override
-        public void createEpochFromConfigUnsafe() { }
-
         @Override
         public TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel)
         {
@@ -110,15 +118,39 @@ public class AccordService implements IAccordService, Shutdownable
             throw new UnsupportedOperationException("Cannot return topology when accord_transactions_enabled = false in cassandra.yaml");
         }
 
+        @Override
+        public void startup() {}
+
         @Override
         public void shutdownAndWait(long timeout, TimeUnit unit) { }
+
+        @Override
+        public Future<Void> epochReady(Epoch epoch)
+        {
+            return BOOTSTRAP_SUCCESS;
+        }
+
+        @Override
+        public void remoteSyncComplete(Message<AccordLocalSyncNotifier.Notification> message) {}
+
+        public boolean isAccordManagedKeyspace(String keyspace)
+        {
+            return false;
+        }
     };
 
+    private static Node.Id localId = null;
     private static class Handle
     {
         public static final AccordService instance = new AccordService();
     }
 
+    public static void startup(NodeId tcmId)
+    {
+        localId = AccordTopologyUtils.tcmIdToAccord(tcmId);
+        instance().startup();
+    }
+
     public static IAccordService instance()
     {
         return DatabaseDescriptor.getAccordTransactionsEnabled() ? Handle.instance : NOOP_SERVICE;
@@ -131,17 +163,18 @@ public class AccordService implements IAccordService, Shutdownable
 
     private AccordService()
     {
-        Node.Id localId = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort());
+        Invariants.checkState(localId != null, "static localId must be set before instantiating AccordService");
         logger.info("Starting accord with nodeId {}", localId);
         AccordAgent agent = new AccordAgent();
-        this.messageSink = new AccordMessageSink(agent);
         this.configService = new AccordConfigurationService(localId);
+        this.messageSink = new AccordMessageSink(agent, configService);
         this.scheduler = new AccordScheduler();
+        this.dataStore = new AccordDataStore();
         this.node = new Node(localId,
                              messageSink,
                              configService,
                              AccordService::uniqueNow,
-                             () -> AccordDataStore.INSTANCE,
+                             () -> dataStore,
                              new KeyspaceSplitter(new EvenSplit<>(DatabaseDescriptor.getAccordShardCount(), getPartitioner().accordSplitter())),
                              agent,
                              new DefaultRandom(),
@@ -150,7 +183,14 @@ public class AccordService implements IAccordService, Shutdownable
                              SimpleProgressLog::new,
                              AccordCommandStores.factory(new AccordJournal().start()));
         this.nodeShutdown = toShutdownable(node);
-        this.verbHandler = new AccordVerbHandler<>(this.node);
+        this.verbHandler = new AccordVerbHandler<>(this.node, configService);
+    }
+
+    @Override
+    public void startup()
+    {
+        configService.start();
+        ClusterMetadataService.instance().log().addListener(configService);
     }
 
     @Override
@@ -159,13 +199,6 @@ public class AccordService implements IAccordService, Shutdownable
         return verbHandler;
     }
 
-    @Override
-    @VisibleForTesting
-    public void createEpochFromConfigUnsafe()
-    {
-        configService.createEpochFromConfig();
-    }
-
     public static long nowInMicros()
     {
         return TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
@@ -248,12 +281,6 @@ public class AccordService implements IAccordService, Shutdownable
                             : new ReadPreemptedException(consistencyLevel, 0, 0, false, txnId.toString());
     }
 
-    @VisibleForTesting
-    AccordMessageSink messageSink()
-    {
-        return messageSink;
-    }
-
     @Override
     public void setCacheSize(long kb)
     {
@@ -308,6 +335,26 @@ public class AccordService implements IAccordService, Shutdownable
         return node;
     }
 
+    @Override
+    public Future<Void> epochReady(Epoch epoch)
+    {
+        AsyncPromise<Void> promise = new AsyncPromise<>();
+        AsyncChain<Void> ready = configService.epochReady(epoch.getEpoch());
+        ready.begin((result, failure) -> {
+            if (failure == null) promise.trySuccess(result);
+            else promise.tryFailure(failure);
+        });
+        return promise;
+    }
+
+    @Override
+    public void remoteSyncComplete(Message<AccordLocalSyncNotifier.Notification> message)
+    {
+        Invariants.checkArgument(localId.equals(message.payload.to), "%s != %s", localId, message.payload.to);
+        configService.remoteSyncComplete(message.payload.from, message.payload.epoch);
+        MessagingService.instance().respond(new AccordLocalSyncNotifier.Acknowledgement(localId), message);
+    }
+
     private static Shutdownable toShutdownable(Node node)
     {
         return new Shutdownable() {
@@ -343,4 +390,15 @@ public class AccordService implements IAccordService, Shutdownable
             }
         };
     }
+
+    @VisibleForTesting
+    public AccordConfigurationService configurationService()
+    {
+        return configService;
+    }
+
+    public boolean isAccordManagedKeyspace(String keyspace)
+    {
+        return configService.isAccordManagedKeyspace(keyspace);
+    }
 }

src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java

@@ -19,84 +19,128 @@
 package org.apache.cassandra.service.accord;
 
 import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import com.google.common.collect.Sets;
+
+import accord.local.Node;
 import accord.topology.Shard;
 import accord.topology.Topology;
-import org.apache.cassandra.db.Keyspace;
+import accord.utils.Invariants;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.locator.EndpointsForRange;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.DistributedSchema;
 import org.apache.cassandra.schema.KeyspaceMetadata;
-import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
 import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
 import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
 
 public class AccordTopologyUtils
 {
-    private static Shard createShard(TokenRange range, EndpointsForToken natural, EndpointsForToken pending)
+    static Node.Id tcmIdToAccord(NodeId nodeId)
     {
-        return new Shard(range,
-                         natural.stream().map(EndpointMapping::getId).collect(Collectors.toList()),
-                         natural.stream().map(EndpointMapping::getId).collect(Collectors.toSet()),
-                         pending.stream().map(EndpointMapping::getId).collect(Collectors.toSet()));
+        return new Node.Id(nodeId.id());
     }
 
-    private static TokenRange minRange(String keyspace, Token token)
+    private static Shard createShard(TokenRange range, Directory directory, EndpointsForRange reads, EndpointsForRange writes)
+    {
+        Function<InetAddressAndPort, Node.Id> endpointMapper = e -> {
+            NodeId tcmId = directory.peerId(e);
+            return tcmIdToAccord(tcmId);
+        };
+        Set<InetAddressAndPort> endpoints = reads.endpoints();
+        Set<InetAddressAndPort> writeEndpoints = writes.endpoints();
+        List<Node.Id> nodes = endpoints.stream().map(endpointMapper).sorted().collect(Collectors.toList());
+        Set<Node.Id> fastPath = new HashSet<>(nodes);  // TODO: support fast path updates
+        Set<Node.Id> pending = endpoints.equals(writeEndpoints) ?
+                               Collections.emptySet() :
+                               writeEndpoints.stream().filter(e -> !endpoints.contains(e)).map(endpointMapper).collect(Collectors.toSet());
+
+        Sets.SetView<InetAddressAndPort> readOnly = Sets.difference(endpoints, writeEndpoints);
+        Invariants.checkState(readOnly.isEmpty(), "Read only replicas detected: %s", readOnly);
+        return new Shard(range, nodes, fastPath, pending);
+    }
+
+    static TokenRange minRange(String keyspace, Token token)
     {
         return new TokenRange(SentinelKey.min(keyspace), new TokenKey(keyspace, token));
     }
 
-    private static TokenRange maxRange(String keyspace, Token token)
+    static TokenRange maxRange(String keyspace, Token token)
     {
         return new TokenRange(new TokenKey(keyspace, token), SentinelKey.max(keyspace));
     }
 
-    private static TokenRange range(String keyspace, Token left, Token right)
+    static TokenRange fullRange(String keyspace)
     {
-        return new TokenRange(new TokenKey(keyspace, left), new TokenKey(keyspace, right));
+        return new TokenRange(SentinelKey.min(keyspace), SentinelKey.max(keyspace));
     }
 
-    public static List<Shard> createShards(String keyspace, ClusterMetadata clusterMetadata)
+    static TokenRange range(String keyspace, Range<Token> range)
     {
-        KeyspaceMetadata keyspaceMetadata = Keyspace.open(keyspace).getMetadata();
-        List<Token> tokens = new ArrayList<>(clusterMetadata.tokenMap.tokens());
-        tokens.sort(Comparator.naturalOrder());
+        Token minToken = range.left.minValue();
+        return new TokenRange(range.left.equals(minToken) ? SentinelKey.min(keyspace) : new TokenKey(keyspace, range.left),
+                              range.right.equals(minToken) ? SentinelKey.max(keyspace) : new TokenKey(keyspace, range.right));
+    }
 
-        List<Shard> shards = new ArrayList<>(tokens.size() + 1);
-        Shard finalShard = null;
-        for (int i = 0, mi = tokens.size(); i < mi; i++)
+    public static List<Shard> createShards(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory)
+    {
+        ReplicationParams replication = keyspace.params.replication;
+        DataPlacement placement = placements.get(replication);
+
+        List<Range<Token>> ranges = placement.reads.ranges();
+        List<Shard> shards = new ArrayList<>(ranges.size());
+        for (Range<Token> range : ranges)
         {
-            Token token = tokens.get(i);
-            EndpointsForToken natural = clusterMetadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get();
-            EndpointsForToken pending = clusterMetadata.pendingEndpointsFor(keyspaceMetadata, token).get();
-            if (i == 0)
-            {
-                shards.add(createShard(minRange(keyspace, token), natural, pending));
-                finalShard = createShard(maxRange(keyspace, tokens.get(mi - 1)), natural, pending);
-            }
-            else
-            {
-                Token prev = tokens.get(i - 1);
-                shards.add(createShard(range(keyspace, prev, token), natural, pending));
-            }
+            EndpointsForRange reads = placement.reads.forRange(range).get();
+            EndpointsForRange writes = placement.reads.forRange(range).get();
+
+            // TCM doesn't create wrap around ranges
+            Invariants.checkArgument(!range.isWrapAround() || range.right.equals(range.right.minValue()),
+                                     "wrap around range %s found", range);
+            shards.add(createShard(range(keyspace.name, range), directory, reads, writes));
         }
-        shards.add(finalShard);
 
         return shards;
     }
 
-    public static Topology createTopology(long epoch)
+    public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, Predicate<String> keyspacePredicate)
     {
-        List<String> keyspaces = new ArrayList<>(Schema.instance.distributedKeyspaces().names());
-        keyspaces.sort(String::compareTo);
-
         List<Shard> shards = new ArrayList<>();
-        for (String keyspace : keyspaces)
-            shards.addAll(createShards(keyspace, ClusterMetadata.current()));
+        for (KeyspaceMetadata keyspace : schema.getKeyspaces())
+        {
+            if (!keyspacePredicate.test(keyspace.name))
+                continue;
+            shards.addAll(createShards(keyspace, placements, directory));
+        }
+        shards.sort((a, b) -> a.range.compare(b.range));
+        return new Topology(epoch.getEpoch(), shards.toArray(new Shard[0]));
+    }
 
-        return new Topology(epoch, shards.toArray(new Shard[0]));
+    public static EndpointMapping directoryToMapping(long epoch, Directory directory)
+    {
+        EndpointMapping.Builder builder = EndpointMapping.builder(epoch);
+        for (NodeId id : directory.peerIds())
+            builder.add(directory.endpoint(id), tcmIdToAccord(id));
+        return builder.build();
+    }
+
+    public static Topology createAccordTopology(ClusterMetadata metadata, Predicate<String> keyspacePredicate)
+    {
+        return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, keyspacePredicate);
     }
 }

src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java

@@ -33,18 +33,21 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
     private static final Logger logger = LoggerFactory.getLogger(AccordVerbHandler.class);
 
     private final Node node;
+    private final AccordEndpointMapper endpointMapper;
 
-    public AccordVerbHandler(Node node)
+    public AccordVerbHandler(Node node, AccordEndpointMapper endpointMapper)
     {
         this.node = node;
+        this.endpointMapper = endpointMapper;
     }
 
     @Override
     public void doVerb(Message<T> message) throws IOException
     {
+        // TODO (desired): need a non-blocking way to inform CMS of an unknown epoch and add callback to it's receipt
+//        ClusterMetadataService.instance().maybeCatchup(message.epoch());
         logger.debug("Receiving {} from {}", message.payload, message.from());
         T request = message.payload;
-        Node.Id from = EndpointMapping.getId(message.from());
         long knownEpoch = request.knownEpoch();
         if (!node.topology().hasEpoch(knownEpoch))
         {
@@ -52,10 +55,10 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
             long waitForEpoch = request.waitForEpoch();
             if (!node.topology().hasEpoch(waitForEpoch))
             {
-                node.withEpoch(waitForEpoch, () -> request.process(node, from, message));
+                node.withEpoch(waitForEpoch, () -> request.process(node, endpointMapper.mappedId(message.from()), message));
                 return;
             }
         }
-        request.process(node, from, message);
+        request.process(node, endpointMapper.mappedId(message.from()), message);
     }
 }

src/java/org/apache/cassandra/service/accord/EndpointMapping.java

@@ -18,119 +18,70 @@
 
 package org.apache.cassandra.service.accord;
 
-import java.net.Inet4Address;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.primitives.Ints;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.ImmutableBiMap;
 
 import accord.local.Node;
+import accord.utils.Invariants;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.Replica;
 
-public class EndpointMapping
+class EndpointMapping implements AccordEndpointMapper
 {
-    static Node.Id endpointToId(InetAddressAndPort endpoint)
+    public static final EndpointMapping EMPTY = new EndpointMapping(0, ImmutableBiMap.of());
+    private final long epoch;
+    private final ImmutableBiMap<Node.Id, InetAddressAndPort> mapping;
+
+    private EndpointMapping(long epoch,
+                            ImmutableBiMap<Node.Id, InetAddressAndPort> mapping)
     {
-        Preconditions.checkArgument(endpoint.getAddress() instanceof Inet4Address);
-        Inet4Address address = (Inet4Address) endpoint.getAddress();
-        int id = Ints.fromByteArray(address.getAddress());
-        return new Node.Id(id);
+        this.epoch = epoch;
+        this.mapping = mapping;
     }
 
-    static InetAddressAndPort idToEndpoint(Node.Id node)
+    long epoch()
     {
-        byte[] bytes = Ints.toByteArray(node.id);
-        try
+        return epoch;
+    }
+
+    @Override
+    public Node.Id mappedId(InetAddressAndPort endpoint)
+    {
+        return mapping.inverse().get(endpoint);
+    }
+
+    @Override
+    public InetAddressAndPort mappedEndpoint(Node.Id id)
+    {
+        return mapping.get(id);
+    }
+
+    static class Builder
+    {
+        private final long epoch;
+        private final BiMap<Node.Id, InetAddressAndPort> mapping = HashBiMap.create();
+
+        public Builder(long epoch)
         {
-            return InetAddressAndPort.getByAddress(InetAddress.getByAddress(bytes));
+            this.epoch = epoch;
         }
-        catch (UnknownHostException e)
+
+        public Builder add(InetAddressAndPort endpoint, Node.Id id)
         {
-            throw new RuntimeException(e);
+            Invariants.checkArgument(!mapping.containsKey(id), "Mapping already exists for Node.Id %s", id);
+            Invariants.checkArgument(!mapping.containsValue(endpoint), "Mapping already exists for %s", endpoint);
+            mapping.put(id, endpoint);
+            return this;
+        }
+
+        public EndpointMapping build()
+        {
+            return new EndpointMapping(epoch, ImmutableBiMap.copyOf(mapping));
         }
     }
 
-    // TODO: Remove this if its one usage in AccordConfigurationService is removed.
-    public static ImmutableCollection<Node.Id> knownIds()
+    static Builder builder(long epoch)
     {
-        return mapping.endpointToId.values();
-    }
-
-    private static class Mapping
-    {
-        private static final Mapping EMPTY = new Mapping(ImmutableMap.of(), ImmutableMap.of());
-        final ImmutableMap<Node.Id, InetAddressAndPort> idToEndpoint;
-        final ImmutableMap<InetAddressAndPort, Node.Id> endpointToId;
-
-        public Mapping(ImmutableMap<Node.Id, InetAddressAndPort> idToEndpoint,
-                       ImmutableMap<InetAddressAndPort, Node.Id> endpointToId)
-        {
-            this.idToEndpoint = idToEndpoint;
-            this.endpointToId = endpointToId;
-        }
-
-        private static <K, V> ImmutableMap<K, V> put(ImmutableMap<K, V> current, K key, V val)
-        {
-            return ImmutableMap.<K, V>builderWithExpectedSize(current.size() + 1).putAll(current).put(key, val).build();
-        }
-
-        public Mapping add(InetAddressAndPort endpoint)
-        {
-            if (endpointToId.containsKey(endpoint))
-                return this;
-            Node.Id id = endpointToId(endpoint);
-            return new Mapping(put(idToEndpoint, id, endpoint), put(endpointToId, endpoint, id));
-        }
-
-        public Mapping add(Node.Id id)
-        {
-            if (idToEndpoint.containsKey(id))
-                return this;
-
-            InetAddressAndPort endpoint = idToEndpoint(id);
-            return new Mapping(put(idToEndpoint, id, endpoint), put(endpointToId, endpoint, id));
-        }
-    }
-
-    private static volatile Mapping mapping = Mapping.EMPTY;
-
-    private EndpointMapping() {}
-
-    public static Node.Id getId(InetAddressAndPort endpoint)
-    {
-        Node.Id id = mapping.endpointToId.get(endpoint);
-        if (id == null)
-        {
-            synchronized (EndpointMapping.class)
-            {
-                mapping = mapping.add(endpoint);
-                id = mapping.endpointToId.get(endpoint);
-            }
-        }
-        return id;
-    }
-
-    // FIXME: put this stuff into the configuration service, where it will eventually live
-    public static Node.Id getId(Replica replica)
-    {
-        return getId(replica.endpoint());
-    }
-
-    public static InetAddressAndPort getEndpoint(Node.Id id)
-    {
-        InetAddressAndPort endpoint = mapping.idToEndpoint.get(id);
-        if (endpoint == null)
-        {
-            synchronized (EndpointMapping.class)
-            {
-                mapping = mapping.add(id);
-                endpoint = mapping.idToEndpoint.get(id);
-            }
-        }
-        return endpoint;
+        return new Builder(epoch);
     }
 }

src/java/org/apache/cassandra/service/accord/IAccordService.java

@@ -23,7 +23,10 @@ import accord.primitives.Txn;
 import accord.topology.TopologyManager;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
 import org.apache.cassandra.service.accord.txn.TxnData;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.utils.concurrent.Future;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -32,8 +35,6 @@ public interface IAccordService
 {
     IVerbHandler<? extends Request> verbHandler();
 
-    void createEpochFromConfigUnsafe();
-
     TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel);
 
     long currentEpoch();
@@ -42,5 +43,22 @@ public interface IAccordService
 
     TopologyManager topology();
 
+    void startup();
+
     void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException;
+
+    /**
+     * Return a future that will complete once the accord has completed it's local bootstrap process
+     * for any ranges gained in the given epoch
+     */
+    Future<Void> epochReady(Epoch epoch);
+
+    void remoteSyncComplete(Message<AccordLocalSyncNotifier.Notification> message);
+
+    /**
+     * Temporary method to avoid double-streaming keyspaces
+     * @param keyspace
+     * @return
+     */
+    boolean isAccordManagedKeyspace(String keyspace);
 }

src/java/org/apache/cassandra/service/accord/TokenRange.java

@@ -23,6 +23,10 @@ import java.io.IOException;
 import accord.api.RoutingKey;
 import accord.primitives.Range;
 import accord.primitives.Ranges;
+import accord.utils.Invariants;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -34,6 +38,14 @@ public class TokenRange extends Range.EndInclusive
     public TokenRange(AccordRoutingKey start, AccordRoutingKey end)
     {
         super(start, end);
+        Invariants.checkArgument(start.keyspace().equals(end.keyspace()),
+                                 "Token ranges cannot cover more than one keyspace start:%s, end:%s",
+                                 start, end);
+    }
+
+    public String keyspace()
+    {
+        return ((AccordRoutingKey) start()).keyspace();
     }
 
     public static TokenRange fullRange(String keyspace)
@@ -56,6 +68,16 @@ public class TokenRange extends Range.EndInclusive
         return pick;
     }
 
+    public org.apache.cassandra.dht.Range<Token> toKeyspaceRange ()
+    {
+        IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+        AccordRoutingKey start = (AccordRoutingKey) start();
+        AccordRoutingKey end = (AccordRoutingKey) end();
+        Token left = start instanceof SentinelKey ? partitioner.getMinimumToken() : start.token();
+        Token right = end instanceof SentinelKey ? partitioner.getMinimumToken() : end.token();
+        return new org.apache.cassandra.dht.Range<>(left, right);
+    }
+
     public static final IVersionedSerializer<TokenRange> serializer = new IVersionedSerializer<TokenRange>()
     {
         @Override

src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java

@@ -170,8 +170,7 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
     {
         Invariants.nonNull(throwable);
         if (state.isComplete())
-            throw new IllegalStateException("Unexpected state " + state, throwable);
-
+            return;
         try
         {
             switch (state)

src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java

@@ -22,6 +22,9 @@ import java.io.IOException;
 
 import com.google.common.base.Preconditions;
 
+import accord.api.Query;
+import accord.api.Read;
+import accord.api.Update;
 import accord.local.Node;
 import accord.local.SaveStatus;
 import accord.local.Status;
@@ -43,6 +46,7 @@ import org.apache.cassandra.service.accord.txn.TxnQuery;
 import org.apache.cassandra.service.accord.txn.TxnRead;
 import org.apache.cassandra.service.accord.txn.TxnUpdate;
 import org.apache.cassandra.service.accord.txn.TxnWrite;
+import org.apache.cassandra.utils.CastingSerializer;
 
 public class CommandSerializers
 {
@@ -118,19 +122,30 @@ public class CommandSerializers
         }
     }
 
-    public static final IVersionedSerializer<PartialTxn> partialTxn = new IVersionedSerializer<PartialTxn>()
+    public static class PartialTxnSerializer implements IVersionedSerializer<PartialTxn>
     {
+        private final IVersionedSerializer<Read> readSerializer;
+        private final IVersionedSerializer<Query> querySerializer;
+        private final IVersionedSerializer<Update> updateSerializer;
+
+        public PartialTxnSerializer(IVersionedSerializer<Read> readSerializer, IVersionedSerializer<Query> querySerializer, IVersionedSerializer<Update> updateSerializer)
+        {
+            this.readSerializer = readSerializer;
+            this.querySerializer = querySerializer;
+            this.updateSerializer = updateSerializer;
+        }
+
         @Override
         public void serialize(PartialTxn txn, DataOutputPlus out, int version) throws IOException
         {
             CommandSerializers.kind.serialize(txn.kind(), out, version);
             KeySerializers.ranges.serialize(txn.covering(), out, version);
             KeySerializers.seekables.serialize(txn.keys(), out, version);
-            TxnRead.serializer.serialize((TxnRead) txn.read(), out, version);
-            TxnQuery.serializer.serialize((TxnQuery) txn.query(), out, version);
+            readSerializer.serialize(txn.read(), out, version);
+            querySerializer.serialize(txn.query(), out, version);
             out.writeBoolean(txn.update() != null);
             if (txn.update() != null)
-                TxnUpdate.serializer.serialize((TxnUpdate) txn.update(), out, version);
+                updateSerializer.serialize(txn.update(), out, version);
         }
 
         @Override
@@ -139,9 +154,9 @@ public class CommandSerializers
             Txn.Kind kind = CommandSerializers.kind.deserialize(in, version);
             Ranges covering = KeySerializers.ranges.deserialize(in, version);
             Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
-            TxnRead read = TxnRead.serializer.deserialize(in, version);
-            TxnQuery query = TxnQuery.serializer.deserialize(in, version);
-            TxnUpdate update = in.readBoolean() ? TxnUpdate.serializer.deserialize(in, version) : null;
+            Read read = readSerializer.deserialize(in, version);
+            Query query = querySerializer.deserialize(in, version);
+            Update update = in.readBoolean() ? updateSerializer.deserialize(in, version) : null;
             return new PartialTxn.InMemory(covering, kind, keys, read, query, update);
         }
 
@@ -151,14 +166,20 @@ public class CommandSerializers
             long size = CommandSerializers.kind.serializedSize(txn.kind(), version);
             size += KeySerializers.ranges.serializedSize(txn.covering(), version);
             size += KeySerializers.seekables.serializedSize(txn.keys(), version);
-            size += TxnRead.serializer.serializedSize((TxnRead) txn.read(), version);
-            size += TxnQuery.serializer.serializedSize((TxnQuery) txn.query(), version);
+            size += readSerializer.serializedSize(txn.read(), version);
+            size += querySerializer.serializedSize(txn.query(), version);
             size += TypeSizes.sizeof(txn.update() != null);
             if (txn.update() != null)
-                size += TxnUpdate.serializer.serializedSize((TxnUpdate) txn.update(), version);
+                size += updateSerializer.serializedSize(txn.update(), version);
             return size;
         }
-    };
+    }
+
+    private static final IVersionedSerializer<Read> read = new CastingSerializer<>(TxnRead.class, TxnRead.serializer);
+    private static final IVersionedSerializer<Query> query = new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer);
+    private static final IVersionedSerializer<Update> update = new CastingSerializer<>(TxnUpdate.class, TxnUpdate.serializer);
+
+    public static final IVersionedSerializer<PartialTxn> partialTxn = new PartialTxnSerializer(read, query, update);
 
     public static final IVersionedSerializer<SaveStatus> saveStatus = new EnumSerializer<>(SaveStatus.class);
     public static final IVersionedSerializer<Status> status = new EnumSerializer<>(Status.class);

src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import accord.api.RoutingKey;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.ReducingRangeMap;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.CollectionSerializers;
+
+public class CommandStoreSerializers
+{
+    private CommandStoreSerializers() {}
+
+    public static IVersionedSerializer<ReducingRangeMap<Timestamp>> rejectBefore = new IVersionedSerializer<ReducingRangeMap<Timestamp>>()
+    {
+        public void serialize(ReducingRangeMap<Timestamp> map, DataOutputPlus out, int version) throws IOException
+        {
+            out.writeBoolean(map.inclusiveEnds());
+            int size = map.size();
+            out.writeUnsignedVInt32(size);
+
+            for (int i=0; i<size; i++)
+            {
+                KeySerializers.routingKey.serialize(map.key(i), out, version);
+                CommandSerializers.timestamp.serialize(map.value(i), out, version);
+            }
+            CommandSerializers.timestamp.serialize(map.value(size), out, version);
+        }
+
+        public ReducingRangeMap<Timestamp> deserialize(DataInputPlus in, int version) throws IOException
+        {
+            boolean inclusiveEnds = in.readBoolean();
+            int size = in.readUnsignedVInt32();
+            RoutingKey[] keys = new RoutingKey[size];
+            Timestamp[] values = new Timestamp[size + 1];
+            for (int i=0; i<size; i++)
+            {
+                keys[i] = KeySerializers.routingKey.deserialize(in, version);
+                values[i] = CommandSerializers.timestamp.deserialize(in, version);
+            }
+            values[size] = CommandSerializers.timestamp.deserialize(in, version);
+            return ReducingRangeMap.SerializerSupport.create(inclusiveEnds, keys, values);
+        }
+
+        public long serializedSize(ReducingRangeMap<Timestamp> map, int version)
+        {
+            long size = TypeSizes.BOOL_SIZE;
+            size += TypeSizes.sizeofUnsignedVInt(size);
+            int mapSize = map.size();
+            for (int i=0; i<mapSize; i++)
+            {
+                size += KeySerializers.routingKey.serializedSize(map.key(i), version);
+                size += CommandSerializers.timestamp.serializedSize(map.value(i), version);
+            }
+            size += CommandSerializers.timestamp.serializedSize(map.value(mapSize), version);
+
+            return size;
+        }
+    };
+
+    private static class TimestampToRangesSerializer<T extends Timestamp> implements IVersionedSerializer<NavigableMap<T, Ranges>>
+    {
+        private final IVersionedSerializer<T> timestampSerializer;
+
+        public TimestampToRangesSerializer(IVersionedSerializer<T> timestampSerializer)
+        {
+            this.timestampSerializer = timestampSerializer;
+        }
+
+        public void serialize(NavigableMap<T, Ranges> map, DataOutputPlus out, int version) throws IOException
+        {
+            CollectionSerializers.serializeMap(map, out, version, timestampSerializer, KeySerializers.ranges);
+        }
+
+        public NavigableMap<T, Ranges> deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return CollectionSerializers.deserializeMap(in, version, timestampSerializer, KeySerializers.ranges, i -> new TreeMap<>());
+
+        }
+
+        public long serializedSize(NavigableMap<T, Ranges> map, int version)
+        {
+            return CollectionSerializers.serializedMapSize(map, version, timestampSerializer, KeySerializers.ranges);
+        }
+    }
+
+    public static final IVersionedSerializer<NavigableMap<TxnId, Ranges>> bootstrapBeganAt = new TimestampToRangesSerializer<>(CommandSerializers.txnId);
+    public static final IVersionedSerializer<NavigableMap<Timestamp, Ranges>> safeToRead = new TimestampToRangesSerializer<>(CommandSerializers.timestamp);
+}

src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord.serializers;
+
+import java.io.IOException;
+
+import accord.api.Data;
+import accord.impl.AbstractFetchCoordinator.FetchRequest;
+import accord.impl.AbstractFetchCoordinator.FetchResponse;
+import accord.local.Status;
+import accord.messages.ReadData;
+import accord.messages.ReadData.ReadReply;
+import accord.primitives.Ranges;
+import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
+import accord.utils.Invariants;
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.AccordFetchCoordinator.StreamData;
+import org.apache.cassandra.service.accord.AccordFetchCoordinator.StreamingTxn;
+import org.apache.cassandra.utils.CastingSerializer;
+
+import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable;
+import static org.apache.cassandra.utils.NullableSerializer.serializeNullable;
+import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize;
+
+public class FetchSerializers
+{
+    public static final IVersionedSerializer<FetchRequest> request = new IVersionedSerializer<FetchRequest>()
+    {
+        @Override
+        public void serialize(FetchRequest request, DataOutputPlus out, int version) throws IOException
+        {
+            Invariants.checkArgument(request.txnId.equals(TxnId.NONE));
+            Invariants.checkArgument(request.waitForStatus == Status.Applied);
+            Invariants.checkArgument(request.waitUntil.equals(Timestamp.MAX));
+
+            out.writeUnsignedVInt(request.waitForEpoch());
+            CommandSerializers.txnId.serialize((TxnId) request.executeReadAt, out, version);
+            KeySerializers.ranges.serialize((Ranges) request.readScope, out, version);
+
+            DepsSerializer.partialDeps.serialize(request.partialDeps, out, version);
+            StreamingTxn.serializer.serialize(request.read, out, version);
+        }
+
+        @Override
+        public FetchRequest deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return new FetchRequest(in.readUnsignedVInt(),
+                                    CommandSerializers.txnId.deserialize(in, version),
+                                    KeySerializers.ranges.deserialize(in, version),
+                                    DepsSerializer.partialDeps.deserialize(in, version),
+                                    StreamingTxn.serializer.deserialize(in, version));
+        }
+
+        @Override
+        public long serializedSize(FetchRequest request, int version)
+        {
+            return TypeSizes.sizeofUnsignedVInt(request.waitForEpoch())
+                   + CommandSerializers.txnId.serializedSize((TxnId) request.executeReadAt, version)
+                   + KeySerializers.ranges.serializedSize((Ranges) request.readScope, version)
+                   + DepsSerializer.partialDeps.serializedSize(request.partialDeps, version)
+                   + StreamingTxn.serializer.serializedSize(request.read, version);
+        }
+    };
+
+    public static final IVersionedSerializer<ReadReply> reply = new IVersionedSerializer<ReadReply>()
+    {
+        final ReadData.ReadNack[] nacks = ReadData.ReadNack.values();
+        final IVersionedSerializer<Data> streamDataSerializer = new CastingSerializer<>(StreamData.class, StreamData.serializer);
+
+        @Override
+        public void serialize(ReadReply reply, DataOutputPlus out, int version) throws IOException
+        {
+            if (!reply.isOk())
+            {
+                out.writeByte(1 + ((ReadData.ReadNack) reply).ordinal());
+                return;
+            }
+
+            out.writeByte(0);
+            FetchResponse response = (FetchResponse) reply;
+            serializeNullable(response.unavailable, out, version, KeySerializers.ranges);
+            serializeNullable(response.data, out, version, streamDataSerializer);
+            CommandSerializers.timestamp.serialize(response.maxApplied, out, version);
+        }
+
+        @Override
+        public ReadReply deserialize(DataInputPlus in, int version) throws IOException
+        {
+            int id = in.readByte();
+            if (id != 0)
+                return nacks[id - 1];
+
+            return new FetchResponse(deserializeNullable(in, version, KeySerializers.ranges),
+                                     deserializeNullable(in, version, streamDataSerializer),
+                                     CommandSerializers.timestamp.deserialize(in, version));
+        }
+
+        @Override
+        public long serializedSize(ReadReply reply, int version)
+        {
+            if (!reply.isOk())
+                return TypeSizes.BYTE_SIZE;
+
+            FetchResponse response = (FetchResponse) reply;
+            return TypeSizes.BYTE_SIZE
+                   + serializedNullableSize(response.unavailable, version, KeySerializers.ranges)
+                   + serializedNullableSize(response.data, version, streamDataSerializer)
+                   + CommandSerializers.timestamp.serializedSize(response.maxApplied, version);
+        }
+    };
+}

src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java

@@ -19,13 +19,21 @@
 package org.apache.cassandra.service.accord.serializers;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Set;
 
 import accord.local.Node;
+import accord.primitives.Range;
+import accord.topology.Shard;
+import accord.topology.Topology;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.service.accord.TokenRange;
+import org.apache.cassandra.utils.ArraySerializers;
+import org.apache.cassandra.utils.CollectionSerializers;
 
 public class TopologySerializers
 {
@@ -69,4 +77,64 @@ public class TopologySerializers
             return TypeSizes.INT_SIZE;  // id.id
         }
     };
+
+    public static final IVersionedSerializer<Shard> shard = new IVersionedSerializer<Shard>()
+    {
+        @Override
+        public void serialize(Shard shard, DataOutputPlus out, int version) throws IOException
+        {
+            TokenRange.serializer.serialize((TokenRange) shard.range, out, version);
+            CollectionSerializers.serializeList(shard.nodes, out, version, nodeId);
+            CollectionSerializers.serializeCollection(shard.fastPathElectorate, out, version, nodeId);
+            CollectionSerializers.serializeCollection(shard.joining, out, version, nodeId);
+
+        }
+
+        @Override
+        public Shard deserialize(DataInputPlus in, int version) throws IOException
+        {
+            Range range = TokenRange.serializer.deserialize(in, version);
+            List<Node.Id> nodes = CollectionSerializers.deserializeList(in, version, nodeId);
+            Set<Node.Id> fastPathElectorate = CollectionSerializers.deserializeSet(in, version, nodeId);
+            Set<Node.Id> joining = CollectionSerializers.deserializeSet(in, version, nodeId);
+            return new Shard(range, nodes, fastPathElectorate, joining);
+        }
+
+        @Override
+        public long serializedSize(Shard shard, int version)
+        {
+            long size = TokenRange.serializer.serializedSize((TokenRange) shard.range, version);
+            size += CollectionSerializers.serializedListSize(shard.nodes, version, nodeId);
+            size += CollectionSerializers.serializedCollectionSize(shard.fastPathElectorate, version, nodeId);
+            size += CollectionSerializers.serializedCollectionSize(shard.joining, version, nodeId);
+            return size;
+        }
+    };
+
+    public static final IVersionedSerializer<Topology> topology = new IVersionedSerializer<Topology>()
+    {
+        @Override
+        public void serialize(Topology topology, DataOutputPlus out, int version) throws IOException
+        {
+            out.writeLong(topology.epoch());
+            ArraySerializers.serializeArray(topology.unsafeGetShards(), out, version, shard);
+        }
+
+        @Override
+        public Topology deserialize(DataInputPlus in, int version) throws IOException
+        {
+            long epoch = in.readLong();
+            Shard[] shards = ArraySerializers.deserializeArray(in, version, shard, Shard[]::new);
+            return new Topology(epoch, shards);
+        }
+
+        @Override
+        public long serializedSize(Topology topology, int version)
+        {
+            long size = 0;
+            size += TypeSizes.LONG_SIZE; // epoch
+            size += ArraySerializers.serializedArraySize(topology.unsafeGetShards(), version, shard);
+            return size;
+        }
+    };
 }

src/java/org/apache/cassandra/tcm/ClusterMetadata.java

@@ -434,6 +434,12 @@ public class ClusterMetadata
             return this;
         }
 
+        public Transformer register(NodeId nodeId, NodeAddresses addresses, Location location, NodeVersion version)
+        {
+            directory = directory.with(nodeId, addresses, location, version);
+            return this;
+        }
+
         public Transformer withNodeState(NodeId id, NodeState state)
         {
             directory = directory.withNodeState(id, state);

src/java/org/apache/cassandra/tcm/Startup.java

@@ -53,6 +53,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.tcm.log.LocalLog;
 import org.apache.cassandra.tcm.log.LogStorage;
 import org.apache.cassandra.tcm.log.SystemKeyspaceStorage;
@@ -415,6 +416,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
     {
         ClusterMetadata metadata = ClusterMetadata.current();
         NodeId self = metadata.myNodeId();
+        AccordService.startup(self);
 
         // finish in-progress sequences first
         InProgressSequences.finishInProgressSequences(self, true);

src/java/org/apache/cassandra/tcm/membership/Directory.java

@@ -168,6 +168,11 @@ public class Directory implements MetadataValue<Directory>
         return with(addresses, location, CURRENT);
     }
 
+    public Directory with(NodeId id, NodeAddresses addresses, Location location, NodeVersion nodeVersion)
+    {
+        return with(addresses, id, id.toUUID(), location, nodeVersion);
+    }
+
     public Directory with(NodeAddresses addresses, Location location, NodeVersion nodeVersion)
     {
         NodeId id = new NodeId(nextId);

src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java

@@ -25,6 +25,7 @@ import java.util.Set;
 import java.util.stream.StreamSupport;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,6 +42,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.streaming.StreamState;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
@@ -60,6 +62,7 @@ import org.apache.cassandra.tcm.transformations.PrepareJoin;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static com.google.common.collect.ImmutableList.of;
@@ -356,12 +359,14 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
 
         StorageService.instance.repairPaxosForTopologyChange("bootstrap");
         Future<StreamState> bootstrapStream = StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements);
+        Future<Void> accordReady = AccordService.instance().epochReady(metadata.epoch);
+        Future<?> ready = FutureCombiner.allOf(Lists.newArrayList(bootstrapStream, accordReady));
         try
         {
             if (bootstrapTimeoutMillis > 0)
-                bootstrapStream.get(bootstrapTimeoutMillis, MILLISECONDS);
+                ready.get(bootstrapTimeoutMillis, MILLISECONDS);
             else
-                bootstrapStream.get();
+                ready.get();
             StorageService.instance.markViewsAsBuilt();
             StorageService.instance.clearOngoingBootstrap();
             logger.info("Bootstrap completed for tokens {}", tokens);

src/java/org/apache/cassandra/tcm/sequences/Move.java

@@ -49,8 +49,10 @@ import org.apache.cassandra.schema.Keyspaces;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.streaming.StreamOperation;
 import org.apache.cassandra.streaming.StreamPlan;
+import org.apache.cassandra.streaming.StreamResultFuture;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
@@ -68,6 +70,8 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.tcm.transformations.PrepareMove;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static com.google.common.collect.ImmutableList.of;
@@ -211,11 +215,12 @@ public class Move extends MultiStepOperation<Epoch>
             case MID_MOVE:
                 try
                 {
+                    ClusterMetadata metadata = ClusterMetadata.current();
                     logger.info("fetching new ranges and streaming old ranges");
                     StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION);
                     Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
                     Map<ReplicationParams, EndpointsByReplica> movementMap = movementMap(FailureDetector.instance,
-                                                                                         ClusterMetadata.current().placements,
+                                                                                         metadata.placements,
                                                                                          toSplitRanges,
                                                                                          startMove.delta(),
                                                                                          midMove.delta(),
@@ -224,6 +229,8 @@ public class Move extends MultiStepOperation<Epoch>
 
                     for (KeyspaceMetadata ks : keyspaces)
                     {
+                        if (AccordService.instance().isAccordManagedKeyspace(ks.name))
+                            continue;
                         ReplicationParams replicationParams = ks.params.replication;
                         if (replicationParams.isMeta())
                             continue;
@@ -248,7 +255,9 @@ public class Move extends MultiStepOperation<Epoch>
                         }
                     }
 
-                    streamPlan.execute().get();
+                    StreamResultFuture streamResult = streamPlan.execute();
+                    Future<Void> accordReady = AccordService.instance().epochReady(metadata.epoch);
+                    FutureCombiner.allOf(streamResult, accordReady).get();
                     StorageService.instance.repairPaxosForTopologyChange("move");
                 }
                 catch (InterruptedException e)

src/java/org/apache/cassandra/tools/NodeTool.java

@@ -105,7 +105,6 @@ public class NodeTool
                 Compact.class,
                 CompactionHistory.class,
                 CompactionStats.class,
-                CreateEpochUnsafe.class,
                 DataPaths.class,
                 Decommission.class,
                 Decommission.Abort.class,

src/java/org/apache/cassandra/utils/CastingSerializer.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+
+/**
+ * Utility for serializing/deserializing from/into generic interface fields where we know (and require) the
+ * generic fields to be implementation specific classes
+ * @param <Generic>
+ * @param <Specific>
+ */
+public class CastingSerializer<Generic, Specific extends Generic> implements IVersionedSerializer<Generic>
+{
+    private final Class<Specific> specificClass;
+    private final IVersionedSerializer<Specific> specificSerializer;
+
+    public CastingSerializer(Class<Specific> specificClass, IVersionedSerializer<Specific> specificSerializer)
+    {
+        this.specificClass = specificClass;
+        this.specificSerializer = specificSerializer;
+    }
+
+    @Override
+    public void serialize(Generic generic, DataOutputPlus out, int version) throws IOException
+    {
+        specificSerializer.serialize(specificClass.cast(generic), out, version);
+    }
+
+    @Override
+    public Generic deserialize(DataInputPlus in, int version) throws IOException
+    {
+        Generic result = specificSerializer.deserialize(in, version);
+        if (result != null && !specificClass.isInstance(result))
+            throw new IllegalStateException("Expected instance of " + specificClass.getName());
+        return result;
+    }
+
+    @Override
+    public long serializedSize(Generic generic, int version)
+    {
+        return specificSerializer.serializedSize(specificClass.cast(generic), version);
+    }
+}

src/java/org/apache/cassandra/utils/CollectionSerializers.java

@@ -75,10 +75,10 @@ public class CollectionSerializers
         return deserializeCollection(in, version, serializer, newHashSet());
     }
 
-    public static <K, V> Map<K,V> deserializeMap(DataInputPlus in, int version, IVersionedSerializer<K> keySerializer, IVersionedSerializer<V> valueSerializer, IntFunction<Map<K,V>> factory) throws IOException
+    public static <K, V, M extends Map<K, V>> M deserializeMap(DataInputPlus in, int version, IVersionedSerializer<K> keySerializer, IVersionedSerializer<V> valueSerializer, IntFunction<M> factory) throws IOException
     {
         int size = checkedCast(in.readUnsignedVInt32());
-        Map<K,V> result = factory.apply(size);
+        M result = factory.apply(size);
         while (size-- > 0)
         {
             K key = keySerializer.deserialize(in, version);

src/java/org/apache/cassandra/utils/concurrent/FutureCombiner.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent;
 import io.netty.util.concurrent.GenericFutureListener;
 import io.netty.util.concurrent.GlobalEventExecutor;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -242,6 +243,11 @@ public class FutureCombiner<T> extends AsyncFuture<T>
         return new FutureCombiner<>(futures, () -> futures.stream().map(f -> f.getNow()).collect(Collectors.toList()), FailFastListener::new);
     }
 
+    public static <V> Future<List<V>> allOf(io.netty.util.concurrent.Future<? extends V>... futures)
+    {
+        return allOf(Arrays.asList(futures));
+    }
+
     /**
      * Waits for all futures to complete, returning a list containing values of all successful input futures. This
      * emulates Guava's Futures::successfulAsList in that results will be in the same order as inputs and any

test/distributed/org/apache/cassandra/distributed/impl/Instance.java

@@ -821,6 +821,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
             ClusterMetadataService.instance().processor().fetchLogAndWait();
             NodeId self = Register.maybeRegister();
             RegistrationStatus.instance.onRegistration();
+            AccordService.startup(self);
             boolean joinRing = config.get(Constants.KEY_DTEST_JOIN_RING) == null || (boolean) config.get(Constants.KEY_DTEST_JOIN_RING);
             if (ClusterMetadata.current().directory.peerState(self) != NodeState.JOINED && joinRing)
             {

test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java

@@ -0,0 +1,464 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.accord;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import accord.local.CommandStore;
+import accord.local.PreLoadContext;
+import accord.primitives.Timestamp;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.shared.NetworkTopology;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.service.accord.AccordConfigurationService;
+import org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot;
+import org.apache.cassandra.service.accord.AccordService;
+import org.apache.cassandra.service.accord.api.PartitionKey;
+import org.apache.cassandra.streaming.StreamManager;
+import org.apache.cassandra.streaming.StreamResultFuture;
+import org.apache.cassandra.streaming.StreamSession;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.assertj.core.api.Assertions;
+
+import static accord.utils.async.AsyncChains.awaitUninterruptiblyAndRethrow;
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+public class AccordBootstrapTest extends TestBaseImpl
+{
+    private static DecoratedKey dk(int key)
+    {
+        IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+        return partitioner.decorateKey(ByteBufferUtil.bytes(key));
+    }
+
+    private static PartitionKey pk(int key, String keyspace, String table)
+    {
+        TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
+        return new PartitionKey(keyspace, tid, dk(key));
+    }
+
+    protected void bootstrapAndJoinNode(Cluster cluster)
+    {
+        IInstanceConfig config = cluster.newInstanceConfig();
+        config.set("auto_bootstrap", true);
+        IInvokableInstance newInstance = cluster.bootstrap(config);
+        newInstance.startup(cluster);
+        // todo: re-add once we fix write survey/join ring = false mode
+//        withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000),
+//                     () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster)));
+//        newInstance.nodetoolResult("join").asserts().success();
+        newInstance.nodetoolResult("describecms").asserts().success(); // just make sure we're joined, remove later
+    }
+
+    private static AccordService service()
+    {
+        return (AccordService) AccordService.instance();
+    }
+
+    private static void awaitEpoch(long epoch)
+    {
+        try
+        {
+            boolean completed = service().epochReady(Epoch.create(epoch)).await(60, TimeUnit.SECONDS);
+            Assertions.assertThat(completed)
+                      .describedAs("Epoch %s did not become ready within timeout on %s -> %s",
+                                   epoch, FBUtilities.getBroadcastAddressAndPort(),
+                                   service().configurationService().getEpochSnapshot(epoch))
+                      .isTrue();
+        }
+        catch (InterruptedException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static void awaitLocalSyncNotification(long epoch)
+    {
+        try
+        {
+            AccordConfigurationService configService = service().configurationService();
+            boolean completed = configService.localSyncNotified(epoch).await(5, TimeUnit.SECONDS);
+            Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s",
+                                            epoch, FBUtilities.getBroadcastAddressAndPort()), completed);
+        }
+        catch (InterruptedException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static long maxEpoch(Cluster cluster)
+    {
+        return cluster.stream().mapToLong(node -> node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch())).max().getAsLong();
+    }
+
+    private static class StreamListener implements StreamManager.StreamListener
+    {
+        private static boolean isRegistered = false;
+        private static final StreamListener listener = new StreamListener();
+
+        private final List<StreamResultFuture> registered = new ArrayList<>();
+
+        static synchronized void register()
+        {
+            if (isRegistered)
+                return;
+            StreamManager.instance.addListener(listener);
+            isRegistered = true;
+        }
+
+        public synchronized void onRegister(StreamResultFuture result)
+        {
+            registered.add(result);
+        }
+
+        public synchronized void forSession(Consumer<StreamSession> consumer)
+        {
+            registered.forEach(future -> {
+                future.getCoordinator().getAllStreamSessions().forEach(consumer);
+            });
+        }
+    }
+
+    @Test
+    public void bootstrapTest() throws Throwable
+    {
+
+        int originalNodeCount = 2;
+        int expandedNodeCount = originalNodeCount + 1;
+
+        try (Cluster cluster = Cluster.build().withNodes(originalNodeCount)
+                                      .withoutVNodes()
+                                      .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount))
+                                      .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0"))
+                                      .withConfig(config -> config.with(NETWORK, GOSSIP))
+                                      .start())
+        {
+            long initialMax = maxEpoch(cluster);
+            for (IInvokableInstance node : cluster)
+            {
+
+                node.runOnInstance(() -> {
+                    Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch());
+                    awaitEpoch(initialMax);
+                    AccordConfigurationService configService = service().configurationService();
+                    long minEpoch = configService.minEpoch();
+
+                    Assert.assertEquals(initialMax, configService.maxEpoch());
+
+                    for (long epoch = minEpoch; epoch < initialMax; epoch++)
+                    {
+                        awaitEpoch(epoch);
+                        Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch));
+                    }
+
+                    awaitLocalSyncNotification(initialMax);
+                    Assert.assertEquals(EpochSnapshot.completed(initialMax), configService.getEpochSnapshot(initialMax));
+                });
+            }
+
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(StreamListener::register);
+            }
+
+            cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}");
+            cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))");
+
+            long schemaChangeMax = maxEpoch(cluster);
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(schemaChangeMax));
+                    awaitEpoch(schemaChangeMax);
+                    AccordConfigurationService configService = service().configurationService();
+
+                    for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++)
+                    {
+                        awaitLocalSyncNotification(epoch);
+                        Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch));
+                    }
+                });
+            }
+
+            for (int key = 0; key < 100; key++)
+            {
+                String query = "BEGIN TRANSACTION\n" +
+                               "  LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
+                               "  SELECT row1.v;\n" +
+                               "  IF row1 IS NULL THEN\n" +
+                               "    INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
+                               "  END IF\n" +
+                               "COMMIT TRANSACTION";
+                AccordTestBase.executeWithRetry(cluster, query);
+            }
+
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    Assert.assertTrue(StreamListener.listener.registered.isEmpty());
+                });
+            }
+
+            bootstrapAndJoinNode(cluster);
+            long bootstrapMax = maxEpoch(cluster);
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(bootstrapMax));
+                    Assert.assertEquals(bootstrapMax, ClusterMetadata.current().epoch.getEpoch());
+                    AccordService service = (AccordService) AccordService.instance();
+                    awaitEpoch(bootstrapMax);
+                    AccordConfigurationService configService = service.configurationService();
+
+                    awaitLocalSyncNotification(bootstrapMax);
+                    Assert.assertEquals(EpochSnapshot.completed(bootstrapMax), configService.getEpochSnapshot(bootstrapMax));
+                });
+            }
+
+            InetAddress node3Addr = cluster.get(3).broadcastAddress().getAddress();
+            for (IInvokableInstance node : cluster.get(1, 2))
+            {
+                node.runOnInstance(() -> {
+
+                    StreamListener.listener.forSession(session -> {
+                        Assert.assertEquals(node3Addr, session.peer.getAddress());
+                        Assert.assertEquals(0, session.getNumRequests());
+                        Assert.assertTrue(session.getNumTransfers() > 0);
+                    });
+
+                    awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(safeStore -> {
+                        CommandStore commandStore = safeStore.commandStore();
+                        Assert.assertEquals(0, commandStore.maxBootstrapEpoch());
+                        Assert.assertEquals(Timestamp.NONE, getOnlyElement(commandStore.bootstrapBeganAt().keySet()));
+                        Assert.assertEquals(Timestamp.NONE, getOnlyElement(commandStore.safeToRead().keySet()));
+                    }));
+                });
+            }
+
+            cluster.get(3).runOnInstance(() -> {
+                List<Range<Token>> ranges = StorageService.instance.getLocalRanges("ks");
+                for (int key = 0; key < 100; key++)
+                {
+                    UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key);
+                    PartitionKey partitionKey = pk(key, "ks", "tbl");
+                    if (ranges.stream().anyMatch(range -> range.contains(partitionKey.token())))
+                    {
+                        UntypedResultSet.Row row = getOnlyElement(result);
+                        Assert.assertEquals(key, row.getInt("c"));
+                        Assert.assertEquals(key, row.getInt("v"));
+
+                        awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(safeStore -> {
+                            if (safeStore.ranges().currentRanges().contains(partitionKey))
+                            {
+                                CommandStore commandStore = safeStore.commandStore();
+                                Assert.assertTrue(commandStore.maxBootstrapEpoch() > 0);
+                                Assert.assertFalse(commandStore.bootstrapBeganAt().isEmpty());
+                                Assert.assertFalse(commandStore.safeToRead().isEmpty());
+
+                                Assert.assertEquals(1, commandStore.bootstrapBeganAt().entrySet().stream()
+                                                                   .filter(entry -> entry.getValue().contains(partitionKey))
+                                                                   .map(entry -> {
+                                                                       Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
+                                                                       return entry;
+                                                                   }).count());
+                                Assert.assertEquals(1, commandStore.safeToRead().entrySet().stream()
+                                                                   .filter(entry -> entry.getValue().contains(partitionKey))
+                                                                   .map(entry -> {
+                                                                       Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
+                                                                       return entry;
+                                                                   }).count());
+                            }
+                        }));
+                    }
+                    else
+                    {
+                        Assert.assertTrue(result.isEmpty());
+                    }
+                }
+            });
+        }
+    }
+
+    @Test
+    public void moveTest() throws Throwable
+    {
+        try (Cluster cluster = Cluster.build().withNodes(3)
+                                      .withoutVNodes()
+                                      .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3))
+                                      .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0"))
+                                      .withConfig(config -> config.with(NETWORK, GOSSIP))
+                                      .start())
+        {
+            long initialMax = maxEpoch(cluster);
+            long[] tokens = new long[3];
+            for (int i=0; i<3; i++)
+            {
+                tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens())));
+            }
+
+            for (IInvokableInstance node : cluster)
+            {
+
+                node.runOnInstance(() -> {
+                    Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch());
+                    awaitEpoch(initialMax);
+                    AccordConfigurationService configService = service().configurationService();
+                    long minEpoch = configService.minEpoch();
+
+                    Assert.assertEquals(initialMax, configService.maxEpoch());
+
+                    for (long epoch = minEpoch; epoch < initialMax; epoch++)
+                    {
+                        awaitEpoch(epoch);
+                        Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch));
+                    }
+
+                    awaitLocalSyncNotification(initialMax);
+                    Assert.assertEquals(EpochSnapshot.completed(initialMax), configService.getEpochSnapshot(initialMax));
+                });
+            }
+
+            cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}");
+            cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))");
+
+            long schemaChangeMax = maxEpoch(cluster);
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    Assert.assertEquals(schemaChangeMax, ClusterMetadata.current().epoch.getEpoch());
+                    AccordService service = (AccordService) AccordService.instance();
+                    awaitEpoch(schemaChangeMax);
+                    AccordConfigurationService configService = service.configurationService();
+
+                    for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++)
+                    {
+                        awaitLocalSyncNotification(epoch);
+                        Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch));
+                    }
+                });
+            }
+
+            for (int key = 0; key < 100; key++)
+            {
+                String query = "BEGIN TRANSACTION\n" +
+                               "  LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
+                               "  SELECT row1.v;\n" +
+                               "  IF row1 IS NULL THEN\n" +
+                               "    INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
+                               "  END IF\n" +
+                               "COMMIT TRANSACTION";
+                AccordTestBase.executeWithRetry(cluster, query);
+            }
+
+            long token = ((tokens[1] - tokens[0]) / 2) + tokens[0];
+            long preMove = maxEpoch(cluster);
+
+            cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token)));
+
+            long moveMax = maxEpoch(cluster);
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(moveMax));
+                    Assert.assertEquals(moveMax, ClusterMetadata.current().epoch.getEpoch());
+                    AccordService service = (AccordService) AccordService.instance();
+                    awaitEpoch(moveMax);
+                    AccordConfigurationService configService = service.configurationService();
+
+                    awaitLocalSyncNotification(moveMax);
+                    Assert.assertEquals(EpochSnapshot.completed(moveMax), configService.getEpochSnapshot(moveMax));
+                });
+            }
+
+            for (IInvokableInstance node : cluster)
+            {
+                node.runOnInstance(() -> {
+                    // validate streaming
+                    List<Range<Token>> ranges = StorageService.instance.getLocalRanges("ks");
+                    TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id;
+                    for (int key = 0; key < 100; key++)
+                    {
+                        DecoratedKey dk = dk(key);
+                        UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key);
+                        if (ranges.stream().anyMatch(range -> range.contains(dk.getToken())))
+                        {
+                            UntypedResultSet.Row row = getOnlyElement(result);
+                            Assert.assertEquals(key, row.getInt("c"));
+                            Assert.assertEquals(key, row.getInt("v"));
+
+                            PartitionKey partitionKey = new PartitionKey("ks", tableId, dk);
+
+                            awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(PreLoadContext.contextFor(partitionKey),
+                                                                                          partitionKey.toUnseekable(), moveMax, moveMax,
+                                                                                          safeStore -> {
+                                if (!safeStore.ranges().allAt(preMove).contains(partitionKey))
+                                {
+                                    CommandStore commandStore = safeStore.commandStore();
+                                    Assert.assertTrue(commandStore.maxBootstrapEpoch() > 0);
+                                    Assert.assertFalse(commandStore.bootstrapBeganAt().isEmpty());
+                                    Assert.assertFalse(commandStore.safeToRead().isEmpty());
+
+                                    Assert.assertEquals(1, commandStore.bootstrapBeganAt().entrySet().stream()
+                                                                       .filter(entry -> entry.getValue().contains(partitionKey))
+                                                                       .map(entry -> {
+                                                                           Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
+                                                                           return entry;
+                                                                       }).count());
+                                    Assert.assertEquals(1, commandStore.safeToRead().entrySet().stream()
+                                                                       .filter(entry -> entry.getValue().contains(partitionKey))
+                                                                       .map(entry -> {
+                                                                           Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
+                                                                           return entry;
+                                                                       }).count());
+                                }
+                            }));
+                        }
+                    }
+                });
+            }
+        }
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java

@@ -2235,7 +2235,6 @@ public class AccordCQLTest extends AccordTestBase
         SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 2}");
         SHARED_CLUSTER.schemaChange("CREATE TABLE " + currentTable + "1 (k int, c int, v int, primary key (k, c))");
         SHARED_CLUSTER.schemaChange("CREATE TABLE " + currentTable + "2 (k int, c int, v int, primary key (k, c))");
-        SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe()));
         SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0)));
         SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + currentTable + "1 (k, c, v) VALUES (1, 2, 3);", ConsistencyLevel.ALL);
         SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + currentTable + "2 (k, c, v) VALUES (2, 2, 4);", ConsistencyLevel.ALL);
@@ -2413,7 +2412,6 @@ public class AccordCQLTest extends AccordTestBase
         SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_users ( org_name text, user text, members_version int static, permissions int, PRIMARY KEY (org_name, user) );");
         SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.user_docs ( user text, doc_id int, title text, org_name text, permissions int, PRIMARY KEY (user, doc_id) );");
 
-        SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe()));
         SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0)));
 
         SHARED_CLUSTER.coordinator(1).execute("INSERT INTO demo_ks.org_users (org_name, user, members_version, permissions) VALUES ('demo', 'blake', 5, 777);\n", ConsistencyLevel.ALL);

test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java

@@ -25,6 +25,7 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.apache.cassandra.schema.Schema;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
@@ -38,7 +39,6 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
-import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.utils.AssertionUtils;
 
@@ -71,7 +71,7 @@ public class AccordFeatureFlagTest extends TestBaseImpl
             // The Accord system keyspace should not be present:
             assertEquals("The Accord system keyspace should not exist",
                          Optional.empty(), cluster.get(1).callOnInstance(() -> Schema.instance.localKeyspaces().get(ACCORD_KEYSPACE_NAME)));
-            
+
             // Make sure virtual tables don't exist:
             IIsolatedExecutor.SerializableCallable<Stream<VirtualTable>> hasAccordVirtualTables = 
                     () -> SystemViewsKeyspace.instance.tables().stream().filter(t -> t.getClass().equals(AccordVirtualTables.Epoch.class));

test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java

@@ -117,7 +117,6 @@ public abstract class AccordTestBase extends TestBaseImpl
     {
         for (String ddl : ddls)
             SHARED_CLUSTER.schemaChange(ddl);
-        SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe()));
 
         // Evict commands from the cache immediately to expose problems loading from disk.
         SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0)));
@@ -182,7 +181,7 @@ public abstract class AccordTestBase extends TestBaseImpl
     }
 
     // TODO: Retry on preemption may become unnecessary after the Unified Log is integrated.
-    protected SimpleQueryResult assertRowEqualsWithPreemptedRetry(Cluster cluster, Object[] row, String check, Object... boundValues)
+    protected static SimpleQueryResult assertRowEqualsWithPreemptedRetry(Cluster cluster, Object[] row, String check, Object... boundValues)
     {
         return assertRowWithPreemptedRetry(cluster, QueryResults.builder().row(row).build(), check, boundValues);
     }
@@ -192,7 +191,7 @@ public abstract class AccordTestBase extends TestBaseImpl
         return assertRowWithPreemptedRetry(cluster, QueryResults.builder().build(), check, boundValues);
     }
 
-    private SimpleQueryResult assertRowWithPreemptedRetry(Cluster cluster, SimpleQueryResult expected, String check, Object... boundValues)
+    private static SimpleQueryResult assertRowWithPreemptedRetry(Cluster cluster, SimpleQueryResult expected, String check, Object... boundValues)
     {
         SimpleQueryResult result = executeWithRetry(cluster, check, boundValues);
         QueryResultUtil.assertThat(result).isEqualTo(expected);
@@ -217,7 +216,7 @@ public abstract class AccordTestBase extends TestBaseImpl
         }
     }
 
-    protected static SimpleQueryResult executeWithRetry(Cluster cluster, String check, Object... boundValues)
+    public static SimpleQueryResult executeWithRetry(Cluster cluster, String check, Object... boundValues)
     {
         check = wrapInTxn(check);
 

test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java

@@ -47,7 +47,6 @@ public class NewSchemaTest extends AccordTestBase
             String table = ks + ".tbl" + i;
             SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + ks + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}");
             SHARED_CLUSTER.schemaChange(String.format("CREATE TABLE %s (pk blob primary key)", table));
-            SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe()));
             SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0)));
 
             List<ByteBuffer> keys = tokensToKeys(tokens());

test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java

@@ -43,7 +43,6 @@ import org.apache.cassandra.distributed.api.LogResult;
 import org.apache.cassandra.distributed.impl.FileLogAction;
 import org.apache.cassandra.distributed.impl.Instance;
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.simulator.Action;
 import org.apache.cassandra.simulator.ActionList;
 import org.apache.cassandra.simulator.ActionPlan;
@@ -191,9 +190,7 @@ abstract class AbstractPairOfSequencesPaxosSimulation extends PaxosSimulation
 
         plan = plan.encapsulate(ActionPlan.setUpTearDown(
             ActionList.of(
-                cluster.stream().map(i -> simulated.run("Insert Partitions", i, executeForPrimaryKeys(preInsertStmt(), primaryKeys))),
-                // TODO (now): this is temporary until we have correct epoch handling
-                cluster.stream().map(i -> simulated.run("Create Accord Epoch", i, () -> AccordService.instance().createEpochFromConfigUnsafe()))
+                cluster.stream().map(i -> simulated.run("Insert Partitions", i, executeForPrimaryKeys(preInsertStmt(), primaryKeys)))
             ),
             ActionList.of(
                 cluster.stream().map(i -> checkErrorLogs(i)),

test/unit/org/apache/cassandra/audit/AuditLoggerTest.java

@@ -64,7 +64,6 @@ import org.apache.cassandra.cql3.QueryEvents;
 import org.apache.cassandra.db.ColumnFamilyStoreMBean;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.utils.JMXServerUtils;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_AUTHORIZER;
@@ -444,7 +443,6 @@ public class AuditLoggerTest extends CQLTester
     public void testTransactionAuditing()
     {
         createTable("CREATE TABLE %s (key int PRIMARY KEY, val int)");
-        AccordService.instance().createEpochFromConfigUnsafe();
 
         Session session = sessionNet();
         String fqTableName = KEYSPACE + "." + currentTable();

test/unit/org/apache/cassandra/auth/TxnAuthTest.java

@@ -36,7 +36,6 @@ import org.apache.cassandra.cql3.statements.TransactionStatement;
 import org.apache.cassandra.exceptions.UnauthorizedException;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.QueryState;
-import org.apache.cassandra.service.accord.AccordService;
 import org.apache.cassandra.transport.messages.ResultMessage;
 
 import static org.junit.Assert.assertEquals;
@@ -68,7 +67,6 @@ public class TxnAuthTest extends CQLTester
     public void setUpTest()
     {
         createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY(k))");
-        AccordService.instance().createEpochFromConfigUnsafe();
     }
 
     @Test

test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java

@@ -21,7 +21,6 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.service.accord.AccordService;
 
 import static org.junit.Assert.assertEquals;
 
@@ -97,8 +96,6 @@ public class NodeLocalConsistencyTest extends CQLTester
         createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key))");
         QueryProcessor.process(formatQuery("INSERT INTO %s (key, val) VALUES ('foo', 0)"), NODE_LOCAL);
 
-        AccordService.instance().createEpochFromConfigUnsafe();
-
         String query = "BEGIN TRANSACTION\n" +
                        "  SELECT * FROM %s WHERE key = 'foo';\n" +
                        "COMMIT TRANSACTION";

test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java

@@ -871,8 +871,6 @@ public class PreparedStatementsTest extends CQLTester
 
     private static void updateTxnState()
     {
-        //TODO Remove this method once CEP-21 and CEP-15 integrate
-        AccordService.instance().createEpochFromConfigUnsafe();
         AccordService.instance().setCacheSize(0);
     }
 }

test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java

@@ -42,7 +42,7 @@ import org.awaitility.Awaitility;
 import org.jboss.byteman.contrib.bmunit.BMRule;
 import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
 
-import static org.apache.cassandra.hints.HintsTestUtil.MockFailureDetector;
+import org.apache.cassandra.utils.MockFailureDetector;
 import static org.apache.cassandra.hints.HintsTestUtil.sendHintsAndResponses;
 import static org.junit.Assert.assertEquals;
 
@@ -52,7 +52,7 @@ public class HintServiceBytemanTest
     private static final String KEYSPACE = "hints_service_test";
     private static final String TABLE = "table";
 
-    private final MockFailureDetector failureDetector = new HintsTestUtil.MockFailureDetector();
+    private final MockFailureDetector failureDetector = new MockFailureDetector();
     private static TableMetadata metadata;
 
     @BeforeClass

test/unit/org/apache/cassandra/hints/HintsServiceTest.java

@@ -41,7 +41,7 @@ import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.StorageService;
 
-import static org.apache.cassandra.hints.HintsTestUtil.MockFailureDetector;
+import org.apache.cassandra.utils.MockFailureDetector;
 import static org.apache.cassandra.hints.HintsTestUtil.sendHintsAndResponses;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;

test/unit/org/apache/cassandra/hints/HintsTestUtil.java

@@ -24,9 +24,6 @@ import com.google.common.collect.Iterators;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.partitions.AbstractBTreePartition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.gms.IFailureDetectionEventListener;
-import org.apache.cassandra.gms.IFailureDetector;
-import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MockMessagingService;
 import org.apache.cassandra.net.MockMessagingSpy;
@@ -91,44 +88,4 @@ final class HintsTestUtil
         }
         return spy;
     }
-
-    static class MockFailureDetector implements IFailureDetector
-    {
-        boolean isAlive = true;
-
-        public boolean isAlive(InetAddressAndPort ep)
-        {
-            return isAlive;
-        }
-
-        public void interpret(InetAddressAndPort ep)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public void report(InetAddressAndPort ep)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public void remove(InetAddressAndPort ep)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        public void forceConviction(InetAddressAndPort ep)
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
 }

test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java

@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.service.accord;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import accord.api.ConfigurationService.EpochReady;
+import accord.impl.AbstractConfigurationServiceTest;
+import accord.local.Node;
+import accord.local.Node.Id;
+import accord.topology.Shard;
+import accord.topology.Topology;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.ConnectionType;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageDelivery;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.service.accord.AccordKeyspace.EpochDiskState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.utils.MockFailureDetector;
+import org.apache.cassandra.utils.concurrent.Future;
+
+import static accord.impl.AbstractConfigurationServiceTest.TestListener;
+import static com.google.common.collect.ImmutableSet.of;
+import static java.lang.String.format;
+import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
+import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
+import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
+import static org.apache.cassandra.service.accord.AccordKeyspace.EPOCH_METADATA;
+import static org.apache.cassandra.service.accord.AccordKeyspace.TOPOLOGIES;
+import static org.apache.cassandra.service.accord.AccordKeyspace.loadEpoch;
+
+public class AccordConfigurationServiceTest
+{
+    private static final Id ID1 = new Id(1);
+    private static final Id ID2 = new Id(2);
+    private static final Id ID3 = new Id(3);
+    private static final List<Id> ID_LIST = ImmutableList.of(ID1, ID2, ID3);
+    private static final Set<Id> ID_SET = ImmutableSet.copyOf(ID_LIST);
+    private static final TableId TBL1 = TableId.fromUUID(new UUID(0, 1));
+    private static final TableId TBL2 = TableId.fromUUID(new UUID(0, 2));
+
+    private static EndpointMapping mappingForEpoch(long epoch)
+    {
+        try
+        {
+            EndpointMapping.Builder builder = EndpointMapping.builder(epoch);
+            builder.add(InetAddressAndPort.getByName("127.0.0.1"), ID1);
+            builder.add(InetAddressAndPort.getByName("127.0.0.2"), ID2);
+            builder.add(InetAddressAndPort.getByName("127.0.0.3"), ID3);
+            return builder.build();
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static EndpointMapping mappingForTopology(Topology topology)
+    {
+        try
+        {
+            EndpointMapping.Builder builder = EndpointMapping.builder(topology.epoch());
+            for (Node.Id id : topology.nodes())
+                builder.add(InetAddressAndPort.getByName("127.0.0." + id.id), id);
+            return builder.build();
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static class Messaging implements MessageDelivery
+    {
+        static class Request
+        {
+            final Message<?> message;
+            final InetAddressAndPort to;
+            final RequestCallback<?> callback;
+
+            public Request(Message<?> message, InetAddressAndPort to, RequestCallback<?> callback)
+            {
+                this.message = message;
+                this.to = to;
+                this.callback = callback;
+            }
+        }
+
+        final List<Request> requests = new ArrayList<>();
+
+        @Override
+        public <REQ> void send(Message<REQ> message, InetAddressAndPort to)
+        {
+            requests.add(new Request(message, to, null));
+        }
+
+        @Override
+        public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb)
+        {
+            requests.add(new Request(message, to, cb));
+        }
+
+        @Override
+        public <REQ, RSP> void sendWithCallback(Message<REQ> message, InetAddressAndPort to, RequestCallback<RSP> cb, ConnectionType specifyConnection)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public <REQ, RSP> Future<Message<RSP>> sendWithResult(Message<REQ> message, InetAddressAndPort to)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public <V> void respond(V response, Message<?> message)
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    @BeforeClass
+    public static void beforeClass() throws Throwable
+    {
+        DatabaseDescriptor.daemonInitialization();
+        DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        ServerTestUtils.daemonInitialization();
+        SchemaLoader.prepareServer();
+        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1),
+                                    parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks"));
+    }
+
+    @Before
+    public void setup()
+    {
+        Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStore(TOPOLOGIES).truncateBlocking();
+        Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStore(EPOCH_METADATA).truncateBlocking();
+    }
+
+    @Test
+    public void initialEpochTest() throws Throwable
+    {
+        AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector());
+        Assert.assertEquals(null, AccordKeyspace.loadEpochDiskState());
+        service.start();
+        Assert.assertEquals(null, AccordKeyspace.loadEpochDiskState());
+        Assert.assertTrue(executeInternal(format("SELECT * FROM %s.%s WHERE epoch=1", ACCORD_KEYSPACE_NAME, TOPOLOGIES)).isEmpty());
+
+        Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET));
+        service.reportTopology(topology1);
+        loadEpoch(1, (epoch, topology, syncStatus, pendingSync, remoteSync) -> {
+            Assert.assertEquals(topology1, topology);
+            Assert.assertTrue(remoteSync.isEmpty());
+        });
+        Assert.assertEquals(new EpochDiskState(1, 1), service.diskState());
+
+        service.remoteSyncComplete(ID1, 1);
+        service.remoteSyncComplete(ID2, 1);
+        loadEpoch(1, (epoch, topology, syncStatus, pendingSync, remoteSync) -> {
+            Assert.assertEquals(topology1, topology);
+            Assert.assertEquals(Sets.newHashSet(ID1, ID2), remoteSync);
+        });
+    }
+
+    @Test
+    public void loadTest() throws Throwable
+    {
+        AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector());
+        service.start();
+
+        Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET));
+        service.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1));
+        service.reportTopology(topology1);
+        service.acknowledgeEpoch(EpochReady.done(1));
+        service.remoteSyncComplete(ID1, 1);
+        service.remoteSyncComplete(ID2, 1);
+        service.remoteSyncComplete(ID3, 1);
+
+        Topology topology2 = new Topology(2, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2)));
+        service.reportTopology(topology2);
+        service.acknowledgeEpoch(EpochReady.done(2));
+        service.remoteSyncComplete(ID1, 2);
+
+        Topology topology3 = new Topology(3, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2)));
+        service.reportTopology(topology3);
+        service.acknowledgeEpoch(EpochReady.done(3));
+
+        AccordConfigurationService loaded = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector());
+        loaded.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1));
+        AbstractConfigurationServiceTest.TestListener listener = new AbstractConfigurationServiceTest.TestListener(loaded, true);
+        loaded.registerListener(listener);
+        loaded.start();
+
+        listener.assertNoTruncates();
+        listener.assertTopologiesFor(1L, 2L, 3L);
+        listener.assertTopologyForEpoch(1, topology1);
+        listener.assertTopologyForEpoch(2, topology2);
+        listener.assertTopologyForEpoch(3, topology3);
+        listener.assertSyncsFor(1L, 2L);
+        listener.assertSyncsForEpoch(1, ID1, ID2, ID3);
+        listener.assertSyncsForEpoch(2, ID1);
+    }
+
+    @Test
+    public void truncateTest()
+    {
+        AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector());
+        TestListener serviceListener = new TestListener(service, true);
+        service.registerListener(serviceListener);
+        service.start();
+
+        Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET));
+        service.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1));
+        service.reportTopology(topology1);
+
+        Topology topology2 = new Topology(2, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2)));
+        service.reportTopology(topology2);
+
+        Topology topology3 = new Topology(3, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2)));
+        service.reportTopology(topology3);
+        service.truncateTopologiesUntil(3);
+        Assert.assertEquals(new EpochDiskState(3, 3), service.diskState());
+        serviceListener.assertTruncates(3L);
+
+        AccordConfigurationService loaded = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector());
+        loaded.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1));
+        TestListener loadListener = new TestListener(loaded, true);
+        loaded.registerListener(loadListener);
+        loaded.start();
+        loadListener.assertTopologiesFor(3L);
+    }
+}

test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java

@@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
-
 import org.mockito.Mockito;
 
 import accord.api.Agent;
@@ -29,6 +28,7 @@ import accord.messages.InformOfTxnId;
 import accord.messages.SimpleReply;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessageDelivery;
 import org.apache.cassandra.net.Verb;
@@ -45,8 +45,11 @@ public class AccordMessageSinkTest
     }
 
     @Test
-    public void informOfTxn()
+    public void informOfTxn() throws Throwable
     {
+        Node.Id id = new Node.Id(1);
+        InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0.1");
+        EndpointMapping mapping = EndpointMapping.builder(5).add(endpoint, id).build();
         // There was an issue where the reply was the wrong verb
         // see CASSANDRA-18375
         InformOfTxnId info = Mockito.mock(InformOfTxnId.class);
@@ -54,8 +57,8 @@ public class AccordMessageSinkTest
         SimpleReply reply = SimpleReply.Ok;
 
         MessageDelivery messaging = Mockito.mock(MessageDelivery.class);
-        AccordMessageSink sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging);
-        sink.reply(new Node.Id(1), req, reply);
+        AccordMessageSink sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging, mapping);
+        sink.reply(id, req, reply);
 
         Mockito.verify(messaging).send(Mockito.any(), Mockito.any());
     }

test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java

@@ -83,7 +83,6 @@ import org.apache.cassandra.service.accord.api.PartitionKey;
 import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
 import org.apache.cassandra.service.accord.txn.TxnData;
 import org.apache.cassandra.service.accord.txn.TxnRead;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
@@ -93,11 +92,6 @@ import static java.lang.String.format;
 
 public class AccordTestUtils
 {
-    public static Id localNodeId()
-    {
-        return EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort());
-    }
-
     public static class Commands
     {
         public static Command notWitnessed(TxnId txnId, PartialTxn txn)
@@ -307,7 +301,7 @@ public class AccordTestUtils
     {
         TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
         TokenRange range = TokenRange.fullRange(metadata.keyspace);
-        Node.Id node = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort());
+        Node.Id node = new Id(1);
         Topology topology = new Topology(1, new Shard(range, Lists.newArrayList(node), Sets.newHashSet(node), Collections.emptySet()));
         NodeTimeService time = new NodeTimeService()
         {
@@ -361,7 +355,7 @@ public class AccordTestUtils
     {
         TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
         TokenRange range = TokenRange.fullRange(metadata.keyspace);
-        Node.Id node = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort());
+        Node.Id node = new Id(1);
         Topology topology = new Topology(1, new Shard(range, Lists.newArrayList(node), Sets.newHashSet(node), Collections.emptySet()));
         AccordCommandStore store = createAccordCommandStore(node, now, topology, loadExecutor, saveExecutor);
         store.execute(PreLoadContext.empty(), safeStore -> ((AccordCommandStore)safeStore.commandStore()).setCacheSize(1 << 20));

test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java

@@ -18,54 +18,177 @@
 
 package org.apache.cassandra.service.accord;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import accord.local.Node.Id;
+import accord.topology.Shard;
 import accord.topology.Topology;
-import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.BufferDecoratedKey;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.DistributedSchema;
+import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
-import org.apache.cassandra.schema.Schema;
-import org.apache.cassandra.schema.TableId;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.service.accord.api.PartitionKey;
-import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.membership.NodeAddresses;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.membership.NodeVersion;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+import org.apache.cassandra.tcm.ownership.DataPlacements;
 
 import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
 
 public class AccordTopologyTest
 {
+    private static final Id ID1 = new Id(1);
+    private static final Id ID2 = new Id(2);
+    private static final Id ID3 = new Id(3);
+    private static final List<Id> NODE_LIST = ImmutableList.of(ID1, ID2, ID3);
+    private static final Set<Id> NODE_SET = ImmutableSet.copyOf(NODE_LIST);
+
+    private static final InetAddressAndPort EP1 = ep(1);
+    private static final InetAddressAndPort EP2 = ep(2);
+    private static final InetAddressAndPort EP3 = ep(3);
+
+    private static final IPartitioner partitioner = Murmur3Partitioner.instance;
+    private static Tables tables = null;
+    private static KeyspaceMetadata keyspace = null;
+    private static final Location LOCATION = new Location("DC1", "RACK1");
+
     @BeforeClass
     public static void beforeClass() throws Throwable
     {
         DatabaseDescriptor.daemonInitialization();
         DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
-        SchemaLoader.prepareServer();
-        SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1),
-                                    parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks"));
-        StorageService.instance.initServer();
+        tables = Tables.of(parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks").build());
+        keyspace = KeyspaceMetadata.create("ks", KeyspaceParams.simple(3), tables);
+    }
+
+    private static InetAddressAndPort ep(int i)
+    {
+        try
+        {
+            return InetAddressAndPort.getByAddressOverrideDefaults(InetAddress.getByAddress(new byte[]{127, 0, 0, (byte)i}), 7012);
+        }
+        catch (UnknownHostException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static NodeId nodeId(int id)
+    {
+        return new NodeId(id);
+    }
+
+    private static void addNode(ClusterMetadata.Transformer transformer, int node, Token token)
+    {
+        NodeId nodeId = nodeId(node);
+        InetAddressAndPort ep = ep(node);
+        NodeAddresses addresses = new NodeAddresses(nodeId.toUUID(), ep, ep, ep);
+        transformer.register(nodeId, addresses, LOCATION, NodeVersion.CURRENT);
+        transformer.withNodeState(nodeId, NodeState.JOINED);
+        transformer.proposeToken(nodeId, Collections.singleton(token));
+    }
+
+    private static ClusterMetadata configureCluster(List<Range<Token>> ranges, Keyspaces keyspaces)
+    {
+        assert ranges.size() == 3;
+
+        IPartitioner partitioner = Murmur3Partitioner.instance;
+        ClusterMetadata empty = new ClusterMetadata(partitioner);
+        ClusterMetadata.Transformer transformer = empty.transformer();
+        transformer.with(new DistributedSchema(Keyspaces.of(keyspace)));
+        addNode(transformer, 1, ranges.get(0).right);
+        addNode(transformer, 2, ranges.get(1).right);
+        addNode(transformer, 3, ranges.get(2).right);
+        ClusterMetadata metadata = transformer.build().metadata;
+
+        for (KeyspaceMetadata keyspace : keyspaces)
+        {
+            ReplicationParams replication = keyspace.params.replication;
+            AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication);
+            DataPlacements.Builder placements = metadata.placements.unbuild();
+            DataPlacement placement = strategy.calculateDataPlacement(metadata.epoch, metadata.tokenMap.toRanges(), metadata);
+            placements.with(replication, placement);
+            metadata = transformer.with(placements.build()).build().metadata;
+        }
+
+        return metadata;
+    }
+
+    private static Token token(long t)
+    {
+        return new Murmur3Partitioner.LongToken(t);
+    }
+
+    private static Range<Token> range(Token left, Token right)
+    {
+        return new Range<>(left, right);
+    }
+
+    private static Range<Token> range(long left, long right)
+    {
+        return range(token(left), token(right));
+    }
+
+    /**
+     * Check converter does the right thing if the ring is constructed with min and max tokens
+     */
+    @Test
+    public void minMaxTokens()
+    {
+        List<Range<Token>> ranges = ImmutableList.of(range(partitioner.getMinimumToken(), token(-100)),
+                                                     range(-100, 100),
+                                                     range(token(100), partitioner.getMaximumToken()));
+        Assert.assertEquals(partitioner.getMinimumToken(), ranges.get(0).left);
+        Assert.assertEquals(partitioner.getMaximumToken(), ranges.get(2).right);
+        ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
+
+        Topology topology = AccordTopologyUtils.createAccordTopology(metadata, ks -> true);
+        Topology expected = new Topology(1,
+                                         new Shard(AccordTopologyUtils.minRange("ks", ranges.get(0).right), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.range("ks", ranges.get(1)), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.range("ks", ranges.get(2)), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.maxRange("ks", ranges.get(2).right), NODE_LIST, NODE_SET));
+
+        Assert.assertEquals(expected, topology);
     }
 
     @Test
-    public void minMaxTokenTest()
+    public void wrapAroundRanges()
     {
-        IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
-        Topology topology = AccordTopologyUtils.createTopology(1);
-        Assert.assertNotEquals(0, topology.size());
-        TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id;
-        Token minToken = partitioner.getMinimumToken();
-        Token maxToken = partitioner.getMaximumToken();
+        List<Range<Token>> ranges = ImmutableList.of(range(-100, 0),
+                                                     range(0, 100),
+                                                     range(100, -100));
 
-//        topology.forKey(new AccordKey.TokenKey(tableId, minToken.minKeyBound()));
-        topology.forKey(new PartitionKey("ks", tableId, new BufferDecoratedKey(minToken, ByteBufferUtil.bytes(0))).toUnseekable());
-//        topology.forKey(new AccordKey.TokenKey(tableId, minToken.maxKeyBound()));
-//        topology.forKey(new AccordKey.TokenKey(tableId, maxToken.minKeyBound()));
-        topology.forKey(new PartitionKey("ks", tableId, new BufferDecoratedKey(maxToken, ByteBufferUtil.bytes(0))).toUnseekable());
-//        topology.forKey(new AccordKey.TokenKey(tableId, maxToken.maxKeyBound()));
+        ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
+        Topology topology = AccordTopologyUtils.createAccordTopology(metadata, ks -> true);
+        Topology expected = new Topology(1,
+                                         new Shard(AccordTopologyUtils.minRange("ks", ranges.get(0).left), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.range("ks", ranges.get(0)), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.range("ks", ranges.get(1)), NODE_LIST, NODE_SET),
+                                         new Shard(AccordTopologyUtils.maxRange("ks", ranges.get(2).left), NODE_LIST, NODE_SET));
+
+        Assert.assertEquals(expected, topology);
     }
 }

test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java

@@ -18,25 +18,24 @@
 
 package org.apache.cassandra.service.accord;
 
-import org.junit.Assert;
 import org.junit.Test;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import accord.local.Node;
-import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.utils.CassandraGenerators;
+import org.assertj.core.api.Assertions;
+import static org.quicktheories.QuickTheory.qt;
+
+import org.quicktheories.generators.SourceDSL;
+
 
 public class EndpointMappingTest
 {
-    private static final Logger logger = LoggerFactory.getLogger(EndpointMappingTest.class);
-
     @Test
     public void identityTest() throws Throwable
     {
-        InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0.1");
-        Node.Id id = EndpointMapping.endpointToId(endpoint);
-        Assert.assertEquals(endpoint, EndpointMapping.idToEndpoint(id));
-        logger.info("{} -> {}", endpoint, id);
+        qt().forAll(CassandraGenerators.INET_ADDRESS_AND_PORT_GEN, SourceDSL.integers().between(1, Integer.MAX_VALUE).map(Node.Id::new)).checkAssert((endpoint, id) -> {
+            EndpointMapping mapping = EndpointMapping.builder(1).add(endpoint, id).build();
+            Assertions.assertThat(mapping.mappedEndpoint(id)).isEqualTo(endpoint);
+        });
     }
 }

test/unit/org/apache/cassandra/utils/MockFailureDetector.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import org.apache.cassandra.gms.IFailureDetectionEventListener;
+import org.apache.cassandra.gms.IFailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+
+public class MockFailureDetector implements IFailureDetector
+{
+    public boolean isAlive = true;
+
+    public boolean isAlive(InetAddressAndPort ep)
+    {
+        return isAlive;
+    }
+
+    public void interpret(InetAddressAndPort ep)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void report(InetAddressAndPort ep)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void remove(InetAddressAndPort ep)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    public void forceConviction(InetAddressAndPort ep)
+    {
+        throw new UnsupportedOperationException();
+    }
+}