diff --git a/modules/accord b/modules/accord index 8830d97ba5..03f937175d 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit 8830d97ba517fb2d0f7f22e8e6b886a98839e694 +Subproject commit 03f937175dbcf04243bb0ac48b64746c1a07bc9c diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 365bc98643..d47a651a06 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -236,6 +236,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner ANTICOMPACTION, SCHEMA_CHANGE, OWNED_RANGES_CHANGE, + ACCORD, UNIT_TESTS // explicitly requested flush needed for a test } diff --git a/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java b/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java index 42a518b961..0dba15a421 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java @@ -64,7 +64,6 @@ public class AccordVirtualTables public DataSet data() { IAccordService accord = AccordService.instance(); - accord.createEpochFromConfigUnsafe(); long epoch = accord.currentEpoch(); diff --git a/src/java/org/apache/cassandra/dht/BootStrapper.java b/src/java/org/apache/cassandra/dht/BootStrapper.java index ebf04dddf8..4ec9e4834f 100644 --- a/src/java/org/apache/cassandra/dht/BootStrapper.java +++ b/src/java/org/apache/cassandra/dht/BootStrapper.java @@ -40,6 +40,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; import org.apache.cassandra.streaming.StreamOperation; @@ -136,6 +137,8 @@ public class BootStrapper extends ProgressEventNotifierSupport logger.debug("Schema does not contain any non-local keyspaces to stream on bootstrap"); for (String keyspaceName : nonLocalStrategyKeyspaces) { + if (AccordService.instance().isAccordManagedKeyspace(keyspaceName)) + continue; KeyspaceMetadata ksm = metadata.schema.getKeyspaces().get(keyspaceName).get(); if (ksm.params.replication.isMeta()) continue; diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index 80afb387d9..537b857fce 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -79,10 +79,12 @@ import org.apache.cassandra.schema.SchemaVersionVerbHandler; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.serializers.AcceptSerializers; import org.apache.cassandra.service.accord.serializers.ApplySerializers; +import org.apache.cassandra.service.accord.AccordLocalSyncNotifier; import org.apache.cassandra.service.accord.serializers.BeginInvalidationSerializers; import org.apache.cassandra.service.accord.serializers.CheckStatusSerializers; import org.apache.cassandra.service.accord.serializers.CommitSerializers; import org.apache.cassandra.service.accord.serializers.EnumSerializer; +import org.apache.cassandra.service.accord.serializers.FetchSerializers; import org.apache.cassandra.service.accord.serializers.GetDepsSerializers; import org.apache.cassandra.service.accord.serializers.InformDurableSerializers; import org.apache.cassandra.service.accord.serializers.InformHomeDurableSerializers; @@ -91,6 +93,8 @@ import org.apache.cassandra.service.accord.serializers.PreacceptSerializers; import org.apache.cassandra.service.accord.serializers.ReadDataSerializers; import org.apache.cassandra.service.accord.serializers.RecoverySerializers; import org.apache.cassandra.service.accord.serializers.WaitOnCommitSerializer; +import org.apache.cassandra.service.paxos.Commit; +import org.apache.cassandra.service.paxos.Commit.Agreed; import org.apache.cassandra.service.paxos.PaxosCommit; import org.apache.cassandra.service.paxos.PaxosCommitAndPrepare; import org.apache.cassandra.service.paxos.PaxosPrepare; @@ -117,8 +121,6 @@ import org.apache.cassandra.tcm.serialization.MessageSerializers; import org.apache.cassandra.utils.BooleanSerializer; import org.apache.cassandra.service.EchoVerbHandler; import org.apache.cassandra.service.SnapshotVerbHandler; -import org.apache.cassandra.service.paxos.Commit; -import org.apache.cassandra.service.paxos.Commit.Agreed; import org.apache.cassandra.service.paxos.PrepareResponse; import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler; import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler; @@ -290,6 +292,10 @@ public enum Verb ACCORD_CHECK_STATUS_REQ (140, P2, writeTimeout, IMMEDIATE, () -> CheckStatusSerializers.request, () -> AccordService.instance().verbHandler(), ACCORD_CHECK_STATUS_RSP ), ACCORD_GET_DEPS_RSP (143, P2, writeTimeout, REQUEST_RESPONSE, () -> GetDepsSerializers.reply, RESPONSE_HANDLER ), ACCORD_GET_DEPS_REQ (142, P2, writeTimeout, IMMEDIATE, () -> GetDepsSerializers.request, () -> AccordService.instance().verbHandler(), ACCORD_GET_DEPS_RSP ), + ACCORD_FETCH_DATA_RSP (145, P2, writeTimeout, REQUEST_RESPONSE, () -> FetchSerializers.reply, RESPONSE_HANDLER ), + ACCORD_FETCH_DATA_REQ (144, P2, writeTimeout, IMMEDIATE, () -> FetchSerializers.request, () -> AccordService.instance().verbHandler(), ACCORD_FETCH_DATA_RSP ), + ACCORD_SYNC_NOTIFY_RSP (147, P2, writeTimeout, REQUEST_RESPONSE, () -> AccordLocalSyncNotifier.Acknowledgement.serializer, RESPONSE_HANDLER ), + ACCORD_SYNC_NOTIFY_REQ (146, P2, writeTimeout, IMMEDIATE, () -> AccordLocalSyncNotifier.Notification.serializer, () -> AccordLocalSyncNotifier.verbHandler, ACCORD_SYNC_NOTIFY_RSP), // generic failure response diff --git a/src/java/org/apache/cassandra/service/Rebuild.java b/src/java/org/apache/cassandra/service/Rebuild.java index 421595bab6..673a6ca486 100644 --- a/src/java/org/apache/cassandra/service/Rebuild.java +++ b/src/java/org/apache/cassandra/service/Rebuild.java @@ -44,11 +44,15 @@ import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.MovementMap; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; @@ -119,11 +123,16 @@ public class Rebuild if (keyspace == null) { for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces().names()) + { + if (AccordService.instance().isAccordManagedKeyspace(keyspaceName)) + continue; streamer.addKeyspaceToFetch(keyspaceName); + } } else if (tokens == null) { - streamer.addKeyspaceToFetch(keyspace); + if (!AccordService.instance().isAccordManagedKeyspace(keyspace)) + streamer.addKeyspaceToFetch(keyspace); } else { @@ -150,10 +159,16 @@ public class Rebuild streamer.addSourceFilter(new RangeStreamer.AllowedSourcesFilter(sources)); } - streamer.addKeyspaceToFetch(keyspace); + if (!AccordService.instance().isAccordManagedKeyspace(keyspace)) + streamer.addKeyspaceToFetch(keyspace); } - streamer.fetchAsync().get(); + StreamResultFuture resultFuture = streamer.fetchAsync(); + // wait for result + Future accordReady = AccordService.instance().epochReady(metadata.epoch); + Future ready = FutureCombiner.allOf(resultFuture, accordReady); + // wait for result + ready.get(); } catch (InterruptedException e) { diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 94eb38c96b..c8eb3be597 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -1884,7 +1884,8 @@ public class StorageProxy implements StorageProxyMBean private static PartitionIterator readWithConsensus(SinglePartitionReadCommand.Group group, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) throws InvalidRequestException, UnavailableException, ReadFailureException, ReadTimeoutException { - if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord) + // TCM explicitly relies on paxos and doesn't work with accord + if (DatabaseDescriptor.getLegacyPaxosStrategy() == accord && !group.metadata().keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) { return readWithAccord(group, consistencyLevel); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCallback.java b/src/java/org/apache/cassandra/service/accord/AccordCallback.java index 20ed9fad69..d25d27f593 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCallback.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCallback.java @@ -34,17 +34,19 @@ import org.apache.cassandra.net.RequestCallback; class AccordCallback extends SafeCallback implements RequestCallback { private static final Logger logger = LoggerFactory.getLogger(AccordCallback.class); + private final AccordEndpointMapper endpointMapper; - public AccordCallback(AgentExecutor executor, Callback callback) + public AccordCallback(AgentExecutor executor, Callback callback, AccordEndpointMapper endpointMapper) { super(executor, callback); + this.endpointMapper = endpointMapper; } @Override public void onResponse(Message msg) { logger.debug("Received response {} from {}", msg.payload, msg.from()); - success(EndpointMapping.endpointToId(msg.from()), msg.payload); + success(endpointMapper.mappedId(msg.from()), msg.payload); } private static Throwable convertReason(RequestFailureReason reason) @@ -59,7 +61,7 @@ class AccordCallback extends SafeCallback implements Request { logger.debug("Received failure {} from {} for {}", failureReason, from, this); // TODO (now): we should distinguish timeout failures with some placeholder Exception - failure(EndpointMapping.endpointToId(from), convertReason(failureReason)); + failure(endpointMapper.mappedId(from), convertReason(failureReason)); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 84869c179d..e19f898915 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -65,6 +65,7 @@ import accord.primitives.Seekables; import accord.primitives.Timestamp; import accord.primitives.TxnId; import accord.utils.Invariants; +import accord.utils.ReducingRangeMap; import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import accord.utils.async.Observable; @@ -150,6 +151,16 @@ public class AccordCommandStore extends CommandStore this::loadCommandsForKey, this::saveCommandsForKey, AccordObjectSizes::commandsForKey); + AccordKeyspace.loadCommandStoreMetadata(id, ((rejectBefore, bootstrapBeganAt, safeToRead) -> { + executor.submit(() -> { + if (rejectBefore != null) + super.setRejectBefore(rejectBefore); + if (bootstrapBeganAt != null) + super.setBootstrapBeganAt(bootstrapBeganAt); + if (safeToRead != null) + super.setSafeToRead(safeToRead); + }); + })); executor.execute(() -> CommandStore.register(this)); executor.execute(this::loadRangesToCommands); } @@ -437,4 +448,25 @@ public class AccordCommandStore extends CommandStore { executor.shutdown(); } + + protected void setRejectBefore(ReducingRangeMap newRejectBefore) + { + super.setRejectBefore(newRejectBefore); + // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases + AccordKeyspace.updateRejectBefore(this, newRejectBefore); + } + + protected void setBootstrapBeganAt(NavigableMap newBootstrapBeganAt) + { + super.setBootstrapBeganAt(newBootstrapBeganAt); + // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases + AccordKeyspace.updateBootstrapBeganAt(this, newBootstrapBeganAt); + } + + protected void setSafeToRead(NavigableMap newSafeToRead) + { + super.setSafeToRead(newSafeToRead); + // TODO (required, correctness): rework to persist via journal once available, this can lose updates in some edge cases + AccordKeyspace.updateSafeToRead(this, newSafeToRead); + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 38040b466b..8e9c5beac2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -141,9 +141,9 @@ public class AccordCommandStores extends CommandStores } @Override - public synchronized Supplier updateTopology(Node node, Topology newTopology) + public synchronized Supplier updateTopology(Node node, Topology newTopology, boolean startSync) { - Supplier start = super.updateTopology(node, newTopology); + Supplier start = super.updateTopology(node, newTopology, startSync); return () -> { EpochReady ready = start.get(); ready.metadata.addCallback(() -> { diff --git a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java index 3ab64c9bd0..1a7a512d09 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java @@ -18,98 +18,385 @@ package org.apache.cassandra.service.accord; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; -import javax.annotation.concurrent.GuardedBy; +import javax.annotation.Nullable; -import accord.api.ConfigurationService; +import com.google.common.annotations.VisibleForTesting; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.impl.AbstractConfigurationService; import accord.local.Node; import accord.topology.Topology; +import accord.utils.Invariants; +import accord.utils.async.AsyncResult; +import accord.utils.async.AsyncResults; +import org.agrona.collections.Long2ObjectHashMap; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.accord.AccordKeyspace.EpochDiskState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.listeners.ChangeListener; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; -/** - * Currently a stubbed out config service meant to be triggered from a dtest - */ -public class AccordConfigurationService implements ConfigurationService +// TODO: listen to FailureDetector and rearrange fast path accordingly +public class AccordConfigurationService extends AbstractConfigurationService implements ChangeListener, AccordEndpointMapper, AccordLocalSyncNotifier.Listener { - private final Node.Id localId; - private final List listeners = new CopyOnWriteArrayList<>(); - @GuardedBy("this") - private final List epochs = new ArrayList<>(); + private static final Logger logger = LoggerFactory.getLogger(AccordConfigurationService.class); - public AccordConfigurationService(Node.Id localId) - { - this.localId = localId; - epochs.add(Topology.EMPTY); - } + private final MessageDelivery messagingService; + private final IFailureDetector failureDetector; + private EpochDiskState diskState = EpochDiskState.EMPTY; + private enum State { INITIALIZED, LOADING, STARTED } - @Override - public void registerListener(Listener listener) - { - listeners.add(listener); - } + private State state = State.INITIALIZED; + private volatile EndpointMapping mapping = EndpointMapping.EMPTY; + private final Long2ObjectHashMap syncNotifiers = new Long2ObjectHashMap<>(); - @Override - public synchronized Topology currentTopology() - { - return epochs.get(epochs.size() - 1); - } + public enum SyncStatus { NOT_STARTED, NOTIFYING, COMPLETED } - @Override - public synchronized Topology getTopologyForEpoch(long epoch) + static class EpochState extends AbstractConfigurationService.AbstractEpochState { - return epochs.get((int) epoch); - } + SyncStatus syncStatus = SyncStatus.NOT_STARTED; + protected final AsyncResult.Settable localSyncNotified = AsyncResults.settable(); - @Override - public synchronized void fetchTopologyForEpoch(long epoch) - { - Topology current = currentTopology(); - if (epoch < current.epoch()) - return; - while (current.epoch() < epoch) + public EpochState(long epoch) { - current = AccordTopologyUtils.createTopology(epochs.size()); - unsafeAddEpoch(current); + super(epoch); + } + + void setSyncStatus(SyncStatus status) + { + this.syncStatus = status; + if (status == SyncStatus.COMPLETED) + localSyncNotified.trySuccess(null); + } + + AsyncResult received() + { + return received; + } + + AsyncResult acknowledged() + { + return acknowledged; + } + + @Nullable AsyncResult reads() + { + return reads; + } + + AsyncResult.Settable localSyncNotified() + { + return localSyncNotified; } } - @Override - public void acknowledgeEpoch(EpochReady ready) + static class EpochHistory extends AbstractConfigurationService.AbstractEpochHistory { - Topology acknowledged = getTopologyForEpoch(ready.epoch); - for (Node.Id node : acknowledged.nodes()) + @Override + protected EpochState createEpochState(long epoch) { - if (node.equals(localId)) - continue; - - ready.coordination.addCallback(() -> { - for (Listener listener : listeners) - listener.onEpochSyncComplete(node, ready.epoch); - }); + return new EpochState(epoch); } } - public synchronized void createEpochFromConfig() + public AccordConfigurationService(Node.Id node, MessageDelivery messagingService, IFailureDetector failureDetector) { - Topology current = currentTopology(); - Topology topology = AccordTopologyUtils.createTopology(epochs.size()); - if (current.equals(topology.withEpoch(current.epoch()))) return; - unsafeAddEpoch(topology); + super(node); + this.messagingService = messagingService; + this.failureDetector = failureDetector; } - private void unsafeAddEpoch(Topology topology) + public AccordConfigurationService(Node.Id node) { - epochs.add(topology); - for (Listener listener : listeners) - listener.onTopologyUpdate(topology); + this(node, MessagingService.instance(), FailureDetector.instance); + } - // TODO: This is a hack to enable simplistic cluster reuse for TxnAuthTest, AccordCQLTest, etc. - // Since we don't have a dist sys that sets this up, we have to just lie... - EndpointMapping.knownIds().forEach(id -> { - for (Listener listener : listeners) - listener.onEpochSyncComplete(id, topology.epoch()); + @Override + protected EpochHistory createEpochHistory() + { + return new EpochHistory(); + } + + public synchronized void start() + { + Invariants.checkState(state == State.INITIALIZED, "Expected state to be INITIALIZED but was %s", state); + state = State.LOADING; + updateMapping(ClusterMetadata.current()); + diskState = AccordKeyspace.loadTopologies(((epoch, topology, syncStatus, pendingSyncNotify, remoteSyncComplete) -> { + if (topology != null) + reportTopology(topology, syncStatus == SyncStatus.NOT_STARTED); + + getOrCreateEpochState(epoch).setSyncStatus(syncStatus); + if (syncStatus == SyncStatus.NOTIFYING) + syncNotifiers.put(epoch, new AccordLocalSyncNotifier(epoch, localId, pendingSyncNotify, this, messagingService, failureDetector, this)); + + remoteSyncComplete.forEach(id -> remoteSyncComplete(id, epoch)); + })); + syncNotifiers.values().forEach(AccordLocalSyncNotifier::start); + state = State.STARTED; + } + + @Override + public Node.Id mappedId(InetAddressAndPort endpoint) + { + return Invariants.nonNull(mapping.mappedId(endpoint)); + } + + @Override + public InetAddressAndPort mappedEndpoint(Node.Id id) + { + return Invariants.nonNull(mapping.mappedEndpoint(id)); + } + + @VisibleForTesting + EpochDiskState diskState() + { + return diskState; + } + + @VisibleForTesting + synchronized void updateMapping(EndpointMapping mapping) + { + if (mapping.epoch() > this.mapping.epoch()) + this.mapping = mapping; + } + + synchronized void updateMapping(ClusterMetadata metadata) + { + updateMapping(AccordTopologyUtils.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory)); + } + + private void reportMetadata(ClusterMetadata metadata) + { + Stage.MISC.submit(() -> { + synchronized (AccordConfigurationService.this) + { + logger.info("Reporting metadata for epoch {}", metadata.epoch.getEpoch()); + updateMapping(metadata); + reportTopology(AccordTopologyUtils.createAccordTopology(metadata, this::isAccordManagedKeyspace)); + } }); } + + private void maybeReportMetadata(ClusterMetadata metadata) + { + // don't report metadata until the previous one has been acknowledged + synchronized (this) + { + long epoch = metadata.epoch.getEpoch(); + if (epochs.maxEpoch() == 0) + { + getOrCreateEpochState(epoch); // touch epoch state so subsequent calls see it + reportMetadata(metadata); + return; + } + getOrCreateEpochState(epoch - 1).acknowledged().addCallback(() -> reportMetadata(metadata)); + } + } + + @Override + public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) + { + maybeReportMetadata(prev); + maybeReportMetadata(next); + } + + @Override + protected void fetchTopologyInternal(long epoch) + { + // TODO: need a non-blocking way to inform CMS of an unknown epoch +// ClusterMetadataService.instance().maybeCatchup(Epoch.create(epoch)); + } + + @Override + protected synchronized void localSyncComplete(Topology topology) + { + long epoch = topology.epoch(); + EpochState epochState = getOrCreateEpochState(epoch); + if (epochState.syncStatus != SyncStatus.NOT_STARTED) + return; + + Set pendingNotification = topology.nodes().stream().filter(i -> !localId.equals(i)).collect(Collectors.toSet()); + AccordLocalSyncNotifier notifier = new AccordLocalSyncNotifier(epoch, localId, pendingNotification, this, messagingService, failureDetector, this); + syncNotifiers.put(epoch, notifier); + diskState = AccordKeyspace.setNotifyingLocalSync(epoch, pendingNotification, diskState); + epochState.setSyncStatus(SyncStatus.NOTIFYING); + notifier.start(); + } + + @Override + public long currentEpoch() + { + return super.currentEpoch(); + } + + @Override + public synchronized void onEndpointAck(Node.Id id, long epoch) + { + EpochState epochState = getOrCreateEpochState(epoch); + if (epochState.syncStatus != SyncStatus.NOTIFYING) + return; + diskState = AccordKeyspace.markLocalSyncAck(id, epoch, diskState); + } + + @Override + public synchronized void onComplete(long epoch) + { + EpochState epochState = getOrCreateEpochState(epoch); + epochState.setSyncStatus(SyncStatus.COMPLETED); + diskState = AccordKeyspace.setCompletedLocalSync(epoch, diskState); + } + + @Override + protected synchronized void topologyUpdatePreListenerNotify(Topology topology) + { + if (state == State.STARTED) + diskState = AccordKeyspace.saveTopology(topology, diskState); + } + + @Override + protected void remoteSyncCompletePreListenerNotify(Node.Id node, long epoch) + { + if (state == State.STARTED) + diskState = AccordKeyspace.markRemoteTopologySync(node, epoch, diskState); + } + + @Override + protected void truncateTopologiesPreListenerNotify(long epoch) + { + Invariants.checkState(state == State.STARTED); + } + + @Override + protected void truncateTopologiesPostListenerNotify(long epoch) + { + if (state == State.STARTED) + diskState = AccordKeyspace.truncateTopologyUntil(epoch, diskState); + } + + @VisibleForTesting + public static class EpochSnapshot + { + public enum ResultStatus + { + PENDING, SUCCESS, FAILURE; + + static ResultStatus of (AsyncResult result) + { + if (result == null || !result.isDone()) + return PENDING; + + return result.isSuccess() ? SUCCESS : FAILURE; + } + } + + public final long epoch; + public final SyncStatus syncStatus; + public final ResultStatus received; + public final ResultStatus acknowledged; + public final ResultStatus reads; + + private EpochSnapshot(EpochState state) + { + this.epoch = state.epoch(); + this.syncStatus = state.syncStatus; + this.received = ResultStatus.of(state.received()); + this.acknowledged = ResultStatus.of(state.acknowledged()); + this.reads = ResultStatus.of(state.reads()); + } + + public EpochSnapshot(long epoch, SyncStatus syncStatus, ResultStatus received, ResultStatus acknowledged, ResultStatus reads) + { + this.epoch = epoch; + this.syncStatus = syncStatus; + this.received = received; + this.acknowledged = acknowledged; + this.reads = reads; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EpochSnapshot that = (EpochSnapshot) o; + return epoch == that.epoch && syncStatus == that.syncStatus && received == that.received && acknowledged == that.acknowledged && reads == that.reads; + } + + public int hashCode() + { + return Objects.hash(epoch, syncStatus, received, acknowledged, reads); + } + + public String toString() + { + return "EpochSnapshot{" + + "epoch=" + epoch + + ", syncStatus=" + syncStatus + + ", received=" + received + + ", acknowledged=" + acknowledged + + ", reads=" + reads + + '}'; + } + + public static EpochSnapshot completed(long epoch) + { + return new EpochSnapshot(epoch, SyncStatus.COMPLETED, ResultStatus.SUCCESS, ResultStatus.SUCCESS, ResultStatus.SUCCESS); + } + + public static EpochSnapshot notStarted(long epoch) + { + return new EpochSnapshot(epoch, SyncStatus.NOT_STARTED, ResultStatus.SUCCESS, ResultStatus.SUCCESS, ResultStatus.SUCCESS); + } + } + + @VisibleForTesting + public synchronized EpochSnapshot getEpochSnapshot(long epoch) + { + if (epoch < epochs.minEpoch() || epoch > epochs.maxEpoch()) + return null; + + return new EpochSnapshot(getOrCreateEpochState(epoch)); + } + + @VisibleForTesting + public synchronized long minEpoch() + { + return epochs.minEpoch(); + } + + @VisibleForTesting + public synchronized long maxEpoch() + { + return epochs.maxEpoch(); + } + + @VisibleForTesting + public synchronized Future localSyncNotified(long epoch) + { + AsyncPromise promise = new AsyncPromise<>(); + getOrCreateEpochState(epoch).localSyncNotified().addCallback((result, failure) -> { + if (failure != null) promise.tryFailure(failure); + else promise.trySuccess(result); + }); + return promise; + } + + public boolean isAccordManagedKeyspace(String keyspace) + { + // TODO (required, interop) : replace with schema flag or other mechanism for classifying accord keyspaces + return !SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspace); + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java index b1f191a396..fee2d633a1 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordDataStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordDataStore.java @@ -23,25 +23,14 @@ import accord.local.Node; import accord.local.SafeCommandStore; import accord.primitives.Ranges; import accord.primitives.SyncPoint; -import accord.primitives.Timestamp; -import accord.utils.async.AsyncResults; -public enum AccordDataStore implements DataStore +public class AccordDataStore implements DataStore { - INSTANCE; - @Override public FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback) { - //TODO (implement): do real work - callback.starting(ranges).started(Timestamp.NONE); - callback.fetched(ranges); - return new ImmediateFetchFuture(ranges); - } - - private static class ImmediateFetchFuture extends AsyncResults.SettableResult implements FetchResult - { - ImmediateFetchFuture(Ranges ranges) { setSuccess(ranges); } - @Override public void abort(Ranges abort) { } + AccordFetchCoordinator coordinator = new AccordFetchCoordinator(node, ranges, syncPoint, callback, safeStore.commandStore()); + coordinator.start(); + return coordinator.result(); } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/CreateEpochUnsafe.java b/src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java similarity index 62% rename from src/java/org/apache/cassandra/tools/nodetool/CreateEpochUnsafe.java rename to src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java index c5ea1246d6..16ff04437e 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/CreateEpochUnsafe.java +++ b/src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java @@ -16,18 +16,16 @@ * limitations under the License. */ -package org.apache.cassandra.tools.nodetool; +package org.apache.cassandra.service.accord; -import io.airlift.airline.Command; -import org.apache.cassandra.tools.NodeProbe; -import org.apache.cassandra.tools.NodeTool; +import accord.local.Node; +import org.apache.cassandra.locator.InetAddressAndPort; -@Command(name="createepochunsafe", description = "manually create an Accord epoch from current topology") -public class CreateEpochUnsafe extends NodeTool.NodeToolCmd +/** + * Maps network addresses to accord ids + */ +public interface AccordEndpointMapper { - @Override - protected void execute(NodeProbe probe) - { - throw new UnsupportedOperationException("git rebase to pick up TCM removes this"); - } + Node.Id mappedId(InetAddressAndPort endpoint); + InetAddressAndPort mappedEndpoint(Node.Id id); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java new file mode 100644 index 0000000000..4e841a17ad --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + +import accord.api.Data; +import accord.api.DataStore; +import accord.api.Query; +import accord.api.Read; +import accord.api.Update; +import accord.impl.AbstractFetchCoordinator; +import accord.local.CommandStore; +import accord.local.Node; +import accord.local.SafeCommandStore; +import accord.primitives.PartialTxn; +import accord.primitives.Range; +import accord.primitives.Ranges; +import accord.primitives.Routable; +import accord.primitives.Seekable; +import accord.primitives.Seekables; +import accord.primitives.SyncPoint; +import accord.primitives.Timestamp; +import accord.primitives.Txn; +import accord.utils.Invariants; +import accord.utils.async.AsyncChain; +import accord.utils.async.AsyncChains; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.service.accord.serializers.KeySerializers; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.streaming.StreamCoordinator; +import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; +import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.CastingSerializer; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.TimeUUID; + +import static org.apache.cassandra.utils.CollectionSerializers.deserializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializeMap; +import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize; + +public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamManager.StreamListener +{ + private static final Query noopQuery = (txnId, executeAt, data, read, update) -> null; + + public static class StreamData implements Data + { + public static class SessionInfo + { + final TimeUUID planId; + final boolean hasData; + + public SessionInfo(TimeUUID planId, boolean hasData) + { + this.planId = planId; + this.hasData = hasData; + } + + static final IVersionedSerializer serializer = new IVersionedSerializer() + { + public void serialize(SessionInfo info, DataOutputPlus out, int version) throws IOException + { + TimeUUID.Serializer.instance.serialize(info.planId, out, version); + out.writeBoolean(info.hasData); + + } + + public SessionInfo deserialize(DataInputPlus in, int version) throws IOException + { + return new SessionInfo(TimeUUID.Serializer.instance.deserialize(in, version), in.readBoolean()); + } + + public long serializedSize(SessionInfo info, int version) + { + return TimeUUID.Serializer.instance.serializedSize(info.planId, version) + TypeSizes.BOOL_SIZE; + } + }; + } + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(StreamData data, DataOutputPlus out, int version) throws IOException + { + serializeMap(data.streams, out, version, TokenRange.serializer, SessionInfo.serializer); + } + + @Override + public StreamData deserialize(DataInputPlus in, int version) throws IOException + { + + return new StreamData(ImmutableMap.copyOf(deserializeMap(in, version, + TokenRange.serializer, + SessionInfo.serializer))); + } + + @Override + public long serializedSize(StreamData data, int version) + { + return serializedMapSize(data.streams, version, TokenRange.serializer, SessionInfo.serializer); + } + }; + + private final ImmutableMap streams; + + public StreamData(ImmutableMap streams) + { + this.streams = streams; + } + + public static StreamData of(TokenRange range, TimeUUID streamId, boolean hasData) + { + return new StreamData(ImmutableMap.of(range, new SessionInfo(streamId, hasData))); + } + + @Override + public Data merge(Data data) + { + StreamData that = (StreamData) data; + if (that.streams.keySet().stream().anyMatch(this.streams::containsKey)) + throw new IllegalStateException(String.format("Unable to merge: key found in multiple StreamData %s %s", + this.streams.keySet(), that.streams.keySet())); + Invariants.checkState(!that.streams.keySet().stream().anyMatch(this.streams::containsKey)); + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(this.streams); + builder.putAll(that.streams); + return new StreamData(builder.build()); + } + } + + // needs to be externally synchronized + private class IncomingStream + { + private final TimeUUID planId; + private Range range; + private Node.Id from; + private StreamResultFuture future; + + public IncomingStream(TimeUUID planId) + { + this.planId = planId; + } + + private void rangeReceived(Range range, Node.Id from) + { + Invariants.nonNull(range); + Invariants.nonNull(from); + Invariants.checkState(this.range == null, "range was not null: %s", this.range); + Invariants.checkState(this.from == null, "from was not null: %s", this.from); + this.range = range; + this.from = from; + maybeListen(); + } + + private void futureReceived(StreamResultFuture future) + { + Invariants.nonNull(future); + Invariants.checkState(this.future == null, "future was not null: %s", this.future); + this.future = future; + maybeListen(); + } + + private void maybeListen() + { + if (range == null || future == null) + return; + + Invariants.nonNull(from); + + future.addCallback((state, fail) -> { + if (fail == null) success(from, Ranges.of(range)); + else fail(from, Ranges.of(range), fail); + }, ((AccordCommandStore) commandStore()).executor()); + } + } + + public static class StreamingRead implements Read + { + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(StreamingRead read, DataOutputPlus out, int version) throws IOException + { + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(read.to, out, version); + KeySerializers.ranges.serialize(read.ranges, out, version); + } + + @Override + public StreamingRead deserialize(DataInputPlus in, int version) throws IOException + { + return new StreamingRead(InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version), + KeySerializers.ranges.deserialize(in, version)); + } + + @Override + public long serializedSize(StreamingRead read, int version) + { + return InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(read.to, version) + + KeySerializers.ranges.serializedSize(read.ranges, version); + } + }; + + private final InetAddressAndPort to; + private final Ranges ranges; + + public StreamingRead(InetAddressAndPort to, Ranges ranges) + { + this.to = to; + this.ranges = ranges; + } + + @Override + public Seekables keys() { return ranges; } + + private static boolean hasDataToStream(StreamCoordinator coordinator, InetAddressAndPort to) + { + for (StreamSession session : coordinator.getAllStreamSessions()) + { + if (!session.peer.equals(to)) + continue; + + Invariants.checkState(session.getNumRequests() == 0, "Requested to send data: %s", session); + if (session.getNumTransfers() > 0) + return true; + } + return false; + } + + @Override + public AsyncChain read(Seekable key, Txn.Kind kind, SafeCommandStore commandStore, Timestamp executeAt, DataStore store) + { + try + { + Invariants.checkArgument(key.domain() == Routable.Domain.Range, "Required Range but saw %s: %s", key.domain(), key); + TokenRange range = (TokenRange) key; + + // TODO (correctness): check epoch + // TODO (correctness): handle dropped tables + KeyspaceMetadata ksm = ClusterMetadata.current().schema.getKeyspaceMetadata(range.keyspace()); + Invariants.checkState(ksm != null, "Keyspace %s not found", range.keyspace()); + Invariants.checkState(ksm.tables.size() > 0, "Keyspace '%s' has no tables", range.keyspace()); + + // FIXME: may also be relocation + StreamPlan plan = new StreamPlan(StreamOperation.BOOTSTRAP, 1, false, + null, PreviewKind.NONE).flushBeforeTransfer(true); + + RangesAtEndpoint ranges = RangesAtEndpoint.toDummyList(Collections.singleton(range.toKeyspaceRange())); + ksm.tables.forEach(table -> plan.transferRanges(to, table.keyspace, ranges, table.name)); + StreamResultFuture future = plan.execute(); + return AsyncChains.success(StreamData.of(range, future.planId, hasDataToStream(future.getCoordinator(), to))); + } + catch (Throwable t) + { + return AsyncChains.failure(t); + } + } + + @Override + public Read slice(Ranges ranges) { return new StreamingRead(to, this.ranges.slice(ranges)); } + + @Override + public Read merge(Read other) { throw new UnsupportedOperationException(); } + } + + public static class StreamingTxn + { + private static final IVersionedSerializer read = new CastingSerializer<>(StreamingRead.class, + StreamingRead.serializer); + + private static final IVersionedSerializer query = new IVersionedSerializer() + { + @Override + public void serialize(Query t, DataOutputPlus out, int version) + { + Invariants.checkArgument(t == noopQuery); + } + + @Override + public Query deserialize(DataInputPlus in, int version) + { + return noopQuery; + } + + @Override + public long serializedSize(Query t, int version) + { + Invariants.checkArgument(t == noopQuery); + return 0; + } + }; + + private static final IVersionedSerializer update = new IVersionedSerializer() + { + @Override + public void serialize(Update t, DataOutputPlus out, int version) + { + Invariants.checkArgument(t == null); + } + + @Override + public Update deserialize(DataInputPlus in, int version) + { + return null; + } + + @Override + public long serializedSize(Update t, int version) + { + Invariants.checkArgument(t == null); + return 0; + } + }; + + // TODO (desired): this could be serialized as an InetAddressAndPort and Ranges if we had a special case PartialTxn implementation + public static final IVersionedSerializer serializer = new CommandSerializers.PartialTxnSerializer(read, query, update); + } + + private final Map streams = new HashMap<>(); + + public AccordFetchCoordinator(Node node, Ranges ranges, SyncPoint syncPoint, DataStore.FetchRanges fetchRanges, CommandStore commandStore) + { + super(node, ranges, syncPoint, fetchRanges, commandStore); + } + + @Override + public void start() + { + StreamManager.instance.addListener(this); + super.start(); + } + + private IncomingStream stream(TimeUUID id) + { + return streams.computeIfAbsent(id, IncomingStream::new); + } + + // called from stream thread + @Override + public synchronized void onRegister(StreamResultFuture result) + { + stream(result.planId).futureReceived(result); + } + + protected void onDone(Ranges success, Throwable failure) + { + StreamManager.instance.removeListener(this); + super.onDone(success, failure); + } + + @Override + protected PartialTxn rangeReadTxn(Ranges ranges) + { + StreamingRead read = new StreamingRead(FBUtilities.getBroadcastAddressAndPort(), ranges); + return new PartialTxn.InMemory(ranges, Txn.Kind.Read, ranges, read, noopQuery, null); + } + + @Override + protected synchronized void onReadOk(Node.Id from, CommandStore commandStore, Data data, Ranges received) + { + if (data == null) + return; + + StreamData streamData = (StreamData) data; + streamData.streams.forEach((range, streamInfo) -> { + if (streamInfo.hasData) + { + stream(streamInfo.planId).rangeReceived(range, from); + } + else + { + // if there was no data to stream, no connection is initiated, and we aren't notified via the stream + // listener, so the stream initiator notifies us and we mark it complete here + success(from, Ranges.of(range)); + } + }); + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java index e67d9952ad..b3bf0265d6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java +++ b/src/java/org/apache/cassandra/service/accord/AccordKeyspace.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service.accord; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -31,13 +32,20 @@ import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Executor; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; + +import javax.annotation.Nullable; import com.google.common.collect.ImmutableSet; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.slf4j.Logger; @@ -56,24 +64,33 @@ import accord.local.Status; import accord.primitives.Ballot; import accord.primitives.PartialDeps; import accord.primitives.PartialTxn; +import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.Route; import accord.primitives.Timestamp; import accord.primitives.Txn; import accord.primitives.TxnId; import accord.primitives.Writes; +import accord.topology.Topology; import accord.utils.Invariants; +import accord.utils.ReducingRangeMap; import accord.utils.async.Observable; import org.apache.cassandra.concurrent.DebuggableTask; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.cql3.statements.ModificationStatement; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Columns; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.IMutation; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.RegularAndStaticColumns; @@ -119,19 +136,25 @@ import org.apache.cassandra.schema.Types; import org.apache.cassandra.schema.UserFunctions; import org.apache.cassandra.schema.Views; import org.apache.cassandra.serializers.UUIDSerializer; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus; import org.apache.cassandra.service.accord.api.AccordRoutingKey; import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.serializers.CommandSerializers; +import org.apache.cassandra.service.accord.serializers.CommandStoreSerializers; import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer; import org.apache.cassandra.service.accord.serializers.DepsSerializer; import org.apache.cassandra.service.accord.serializers.KeySerializers; import org.apache.cassandra.service.accord.serializers.ListenerSerializers; +import org.apache.cassandra.service.accord.serializers.TopologySerializers; import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import static java.lang.String.format; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; +import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; import static org.apache.cassandra.db.rows.BufferCell.live; import static org.apache.cassandra.db.rows.BufferCell.tombstone; import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; @@ -144,6 +167,9 @@ public class AccordKeyspace public static final String COMMANDS = "commands"; public static final String COMMANDS_FOR_KEY = "commands_for_key"; + public static final String TOPOLOGIES = "topologies"; + public static final String EPOCH_METADATA = "epoch_metadata"; + public static final String COMMAND_STORE_METADATA = "command_store_metadata"; private static final TupleType TIMESTAMP_TYPE = new TupleType(Lists.newArrayList(LongType.instance, LongType.instance, Int32Type.instance)); private static final String TIMESTAMP_TUPLE = TIMESTAMP_TYPE.asCQL3Type().toString(); @@ -226,7 +252,7 @@ public class AccordKeyspace .build(); // TODO: naming is not very clearly distinct from the base serializers - private static class CommandsSerializers + private static class LocalVersionedSerializers { static final LocalVersionedSerializer> route = localSerializer(KeySerializers.route); static final LocalVersionedSerializer routingKey = localSerializer(AccordRoutingKey.serializer); @@ -235,6 +261,10 @@ public class AccordKeyspace static final LocalVersionedSerializer writes = localSerializer(CommandSerializers.writes); static final LocalVersionedSerializer result = localSerializer(TxnData.serializer); static final LocalVersionedSerializer listeners = localSerializer(ListenerSerializers.listener); + static final LocalVersionedSerializer topology = localSerializer(TopologySerializers.topology); + static final LocalVersionedSerializer> rejectBefore = localSerializer(CommandStoreSerializers.rejectBefore); + static final LocalVersionedSerializer> bootstrapBeganAt = localSerializer(CommandStoreSerializers.bootstrapBeganAt); + static final LocalVersionedSerializer> safeToRead = localSerializer(CommandStoreSerializers.safeToRead); private static LocalVersionedSerializer localSerializer(IVersionedSerializer serializer) { @@ -246,7 +276,7 @@ public class AccordKeyspace { ColumnMetadata column = metadata.getColumn(new ColumnIdentifier(name, true)); if (column == null) - throw new IllegalArgumentException(String.format("Unknown column %s for %s.%s", name, metadata.keyspace, metadata.name)); + throw new IllegalArgumentException(format("Unknown column %s for %s.%s", name, metadata.keyspace, metadata.name)); return column; } @@ -339,6 +369,39 @@ public class AccordKeyspace } } + private static final TableMetadata Topologies = + parse(TOPOLOGIES, + "accord topologies", + "CREATE TABLE %s (" + + "epoch bigint primary key, " + + "topology blob, " + + "sync_state int, " + + "pending_sync_notify set, " + // nodes that need to be told we're synced + "remote_sync_complete set " + // nodes that have told us they're synced + ')').build(); + + private static final TableMetadata EpochMetadata = + parse(EPOCH_METADATA, + "global epoch info", + "CREATE TABLE %s (" + + "key int primary key, " + + "min_epoch bigint, " + + "max_epoch bigint " + + ')').build(); + + private static final TableMetadata CommandStoreMetadata = + parse(COMMAND_STORE_METADATA, + "command store state", + "CREATE TABLE %s (" + + "store_id int, " + + "reject_before blob, " + + "bootstrap_began_at blob, " + + "safe_to_read blob, " + + "PRIMARY KEY(store_id)" + + ')').build(); + + private static final AtomicLong commandStoreMetadataTimestamp = new AtomicLong(); + private static TableMetadata.Builder parse(String name, String description, String cql) { return CreateTableStatement.parse(format(cql, name), ACCORD_KEYSPACE_NAME) @@ -347,6 +410,11 @@ public class AccordKeyspace .gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90)); } + private static void flush(TableMetadata table) + { + Keyspace.open(table.keyspace).getColumnFamilyStore(table.id).forceBlockingFlush(ColumnFamilyStore.FlushReason.ACCORD); + } + public static KeyspaceMetadata metadata() { return KeyspaceMetadata.create(ACCORD_KEYSPACE_NAME, KeyspaceParams.local(), tables(), Views.none(), Types.none(), UserFunctions.none()); @@ -354,7 +422,7 @@ public class AccordKeyspace private static Tables tables() { - return Tables.of(Commands, CommandsForKeys); + return Tables.of(Commands, CommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata); } private static ByteBuffer serialize(T obj, LocalVersionedSerializer serializer) throws IOException @@ -364,7 +432,7 @@ public class AccordKeyspace { serializer.serialize(obj, out); ByteBuffer bb = out.buffer(); - assert size == bb.limit() : String.format("Expected to write %d but wrote %d", size, bb.limit()); + assert size == bb.limit() : format("Expected to write %d but wrote %d", size, bb.limit()); return bb; } } @@ -422,7 +490,7 @@ public class AccordKeyspace Listeners result = new Listeners(); for (ByteBuffer bytes : serialized) { - result.add(deserialize(bytes, CommandsSerializers.listeners)); + result.add(deserialize(bytes, LocalVersionedSerializers.listeners)); } return new Listeners.Immutable(result); } @@ -458,7 +526,7 @@ public class AccordKeyspace private static void addKeyCellIfModified(ColumnMetadata column, Function get, Row.Builder builder, long timestampMicros, C original, C command) throws IOException { - addCellIfModified(column, get, v -> serializeOrNull((AccordRoutingKey) v, CommandsSerializers.routingKey), builder, timestampMicros, original, command); + addCellIfModified(column, get, v -> serializeOrNull((AccordRoutingKey) v, LocalVersionedSerializers.routingKey), builder, timestampMicros, original, command); } private static > void addEnumCellIfModified(ColumnMetadata column, Function get, Row.Builder builder, long timestampMicros, C original, C command) throws IOException @@ -545,17 +613,17 @@ public class AccordKeyspace addEnumCellIfModified(CommandsColumns.status, Command::saveStatus, builder, timestampMicros, original, command); addKeyCellIfModified(CommandsColumns.home_key, Command::homeKey, builder, timestampMicros, original, command); addKeyCellIfModified(CommandsColumns.progress_key, Command::progressKey, builder, timestampMicros, original, command); - addCellIfModified(CommandsColumns.route, Command::route, CommandsSerializers.route, builder, timestampMicros, original, command); + addCellIfModified(CommandsColumns.route, Command::route, LocalVersionedSerializers.route, builder, timestampMicros, original, command); addEnumCellIfModified(CommandsColumns.durability, Command::durability, builder, timestampMicros, original, command); - addCellIfModified(CommandsColumns.txn, Command::partialTxn, CommandsSerializers.partialTxn, builder, timestampMicros, original, command); + addCellIfModified(CommandsColumns.txn, Command::partialTxn, LocalVersionedSerializers.partialTxn, builder, timestampMicros, original, command); addCellIfModified(CommandsColumns.execute_at, Command::executeAt, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command); addCellIfModified(CommandsColumns.promised_ballot, Command::promised, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command); addCellIfModified(CommandsColumns.accepted_ballot, Command::accepted, AccordKeyspace::serializeTimestamp, builder, timestampMicros, original, command); - addCellIfModified(CommandsColumns.dependencies, Command::partialDeps, CommandsSerializers.partialDeps, builder, timestampMicros, original, command); + addCellIfModified(CommandsColumns.dependencies, Command::partialDeps, LocalVersionedSerializers.partialDeps, builder, timestampMicros, original, command); - addSetChanges(CommandsColumns.listeners, Command::durableListeners, v -> serialize(v, CommandsSerializers.listeners), builder, timestampMicros, nowInSeconds, original, command); + addSetChanges(CommandsColumns.listeners, Command::durableListeners, v -> serialize(v, LocalVersionedSerializers.listeners), builder, timestampMicros, nowInSeconds, original, command); if (command.isCommitted()) { @@ -567,8 +635,8 @@ public class AccordKeyspace { Command.Executed executed = command.asExecuted(); Command.Executed originalExecuted = original != null && original.isExecuted() ? original.asExecuted() : null; - addCellIfModified(CommandsColumns.writes, Command.Executed::writes, v -> serialize(v, CommandsSerializers.writes), builder, timestampMicros, originalExecuted, executed); - addCellIfModified(CommandsColumns.result, Command.Executed::result, v -> serialize((TxnData) v, CommandsSerializers.result), builder, timestampMicros, originalExecuted, executed); + addCellIfModified(CommandsColumns.writes, Command.Executed::writes, v -> serialize(v, LocalVersionedSerializers.writes), builder, timestampMicros, originalExecuted, executed); + addCellIfModified(CommandsColumns.result, Command.Executed::result, v -> serialize((TxnData) v, LocalVersionedSerializers.result), builder, timestampMicros, originalExecuted, executed); } } @@ -663,7 +731,7 @@ public class AccordKeyspace "AND domain = ? " + "AND txn_id=(?, ?, ?)"; - return executeInternal(String.format(cql, ACCORD_KEYSPACE_NAME, COMMANDS), + return executeInternal(format(cql, ACCORD_KEYSPACE_NAME, COMMANDS), commandStore.id(), txnId.domain().ordinal(), txnId.msb, txnId.lsb, txnId.node.id); @@ -741,7 +809,7 @@ public class AccordKeyspace @Override public String description() { - return String.format("Table Walker for %s; queries = %d", getClass().getSimpleName(), numQueries); + return format("Table Walker for %s; queries = %d", getClass().getSimpleName(), numQueries); } } @@ -785,7 +853,7 @@ public class AccordKeyspace super(executor, callback); this.storeId = commandStore; this.domain = domain.ordinal(); - cql = String.format("SELECT %s " + + cql = format("SELECT %s " + "FROM %s " + "WHERE store_id = ? " + " AND domain = ? " + @@ -839,14 +907,14 @@ public class AccordKeyspace this.end = end; String selection = selection(CommandsForKeys, requiredColumns, COLUMNS_FOR_ITERATION); - this.cqlFirst = String.format("SELECT DISTINCT %s\n" + + this.cqlFirst = format("SELECT DISTINCT %s\n" + "FROM %s\n" + "WHERE store_id = ?\n" + (startInclusive ? " AND key_token >= ?\n" : " AND key_token > ?\n") + (endInclusive ? " AND key_token <= ?\n" : " AND key_token < ?\n") + "ALLOW FILTERING", selection, CommandsForKeys); - this.cqlContinue = String.format("SELECT DISTINCT %s\n" + + this.cqlContinue = format("SELECT DISTINCT %s\n" + "FROM %s\n" + "WHERE store_id = ?\n" + " AND key_token > ?\n" + @@ -891,9 +959,9 @@ public class AccordKeyspace CommonAttributes.Mutable attributes = new CommonAttributes.Mutable(txnId); // TODO: something less brittle than ordinal, more efficient than values() attributes.durability(Status.Durability.values()[row.getInt("durability", 0)]); - attributes.homeKey(deserializeOrNull(row.getBlob("home_key"), CommandsSerializers.routingKey)); - attributes.progressKey(deserializeOrNull(row.getBlob("progress_key"), CommandsSerializers.routingKey)); - attributes.route(deserializeOrNull(row.getBlob("route"), CommandsSerializers.route)); + attributes.homeKey(deserializeOrNull(row.getBlob("home_key"), LocalVersionedSerializers.routingKey)); + attributes.progressKey(deserializeOrNull(row.getBlob("progress_key"), LocalVersionedSerializers.routingKey)); + attributes.route(deserializeOrNull(row.getBlob("route"), LocalVersionedSerializers.route)); attributes.partialTxn(deserializeTxn(row)); attributes.partialDeps(deserializeDependencies(row)); attributes.setListeners(deserializeListeners(row, "listeners")); @@ -903,8 +971,8 @@ public class AccordKeyspace Ballot accepted = deserializeTimestampOrNull(row, "accepted_ballot", Ballot::fromBits); ImmutableSortedSet waitingOnCommit = deserializeTxnIdNavigableSet(row, "waiting_on_commit"); ImmutableSortedMap waitingOnApply = deserializeWaitingOnApply(row.getMap("waiting_on_apply", BytesType.instance, BytesType.instance)); - Writes writes = deserializeWithVersionOr(row, "writes", CommandsSerializers.writes, () -> null); - Result result = deserializeWithVersionOr(row, "result", CommandsSerializers.result, () -> null); + Writes writes = deserializeWithVersionOr(row, "writes", LocalVersionedSerializers.writes, () -> null); + Result result = deserializeWithVersionOr(row, "result", LocalVersionedSerializers.result, () -> null); switch (status.status) { @@ -941,7 +1009,7 @@ public class AccordKeyspace public static PartialDeps deserializeDependencies(UntypedResultSet.Row row) throws IOException { - return deserializeOrNull(row.getBlob("dependencies"), CommandsSerializers.partialDeps); + return deserializeOrNull(row.getBlob("dependencies"), LocalVersionedSerializers.partialDeps); } public static Timestamp deserializeExecuteAt(UntypedResultSet.Row row) @@ -961,7 +1029,7 @@ public class AccordKeyspace public static PartialTxn deserializeTxn(UntypedResultSet.Row row) throws IOException { - return deserializeOrNull(row.getBlob("txn"), CommandsSerializers.partialTxn); + return deserializeOrNull(row.getBlob("txn"), LocalVersionedSerializers.partialTxn); } public static PartitionKey deserializeKey(UntypedResultSet.Row row) @@ -1094,7 +1162,7 @@ public class AccordKeyspace private static ByteBuffer cellValue(Row row, ColumnMetadata column) { Cell cell = row.getCell(column); - return (cell != null && !cell.isTombstone()) ? cellValue(cell) : null; + return (cell != null && !cell.isTombstone()) ? cellValue(cell) : null; } private static ByteBuffer clusteringValue(Clustering clustering, int idx) @@ -1124,8 +1192,8 @@ public class AccordKeyspace for (SeriesKind kind : SeriesKind.values()) seriesMaps.put(kind, new ImmutableSortedMap.Builder<>(Comparator.naturalOrder())); - try(ReadExecutionController controller = command.executionController(); - FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds)) + try (ReadExecutionController controller = command.executionController(); + FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds)) { if (!partitions.hasNext()) { @@ -1175,4 +1243,297 @@ public class AccordKeyspace throw t; } } + + public static class EpochDiskState + { + public static final EpochDiskState EMPTY = new EpochDiskState(0, 0); + public final long minEpoch; + public final long maxEpoch; + + public EpochDiskState(long minEpoch, long maxEpoch) + { + Invariants.checkArgument(minEpoch >= 0, "Min Epoch %d < 0", minEpoch); + Invariants.checkArgument(maxEpoch >= minEpoch, "Max epoch %d < min %d", maxEpoch, minEpoch); + this.minEpoch = minEpoch; + this.maxEpoch = maxEpoch; + } + + private EpochDiskState withNewMaxEpoch(long epoch) + { + Invariants.checkArgument(epoch > maxEpoch, "Epoch %d <= %d (max)", epoch, maxEpoch); + return new EpochDiskState(Math.max(1, minEpoch), epoch); + } + + private EpochDiskState withNewMinEpoch(long epoch) + { + Invariants.checkArgument(epoch > minEpoch, "epoch %d <= %d (min)", epoch, minEpoch); + Invariants.checkArgument(epoch <= maxEpoch, "epoch %d > %d (max)", epoch, maxEpoch); + return new EpochDiskState(epoch, maxEpoch); + } + + @Override + public String toString() + { + return "EpochDiskState{" + + "minEpoch=" + minEpoch + + ", maxEpoch=" + maxEpoch + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EpochDiskState diskState = (EpochDiskState) o; + return minEpoch == diskState.minEpoch && maxEpoch == diskState.maxEpoch; + } + + @Override + public int hashCode() + { + throw new UnsupportedOperationException(); + } + } + + private static void saveEpochDiskState(EpochDiskState diskState) + { + String cql = "INSERT INTO %s.%s (key, min_epoch, max_epoch) VALUES (0, ?, ?);"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, EPOCH_METADATA), + diskState.minEpoch, diskState.maxEpoch); + } + + @Nullable + @VisibleForTesting + public static EpochDiskState loadEpochDiskState() + { + String cql = "SELECT * FROM %s.%s WHERE key=0"; + UntypedResultSet result = executeInternal(format(cql, ACCORD_KEYSPACE_NAME, EPOCH_METADATA)); + if (result.isEmpty()) + return null; + UntypedResultSet.Row row = result.one(); + return new EpochDiskState(row.getLong("min_epoch"), row.getLong("max_epoch")); + } + + /** + * Update the disk state for this epoch, if it's higher than the one we have one disk. + * + * This is meant to be called before any update involving the new epoch, not after. This way if the update + * fails, we can detect and cleanup. If we updated disk state after an update and it failed, we could "forget" + * about (now acked) topology updates after a restart. + */ + private static EpochDiskState maybeUpdateMaxEpoch(EpochDiskState diskState, long epoch) + { + Invariants.checkArgument(epoch >= diskState.minEpoch, "Epoch %d < %d (min)", epoch, diskState.minEpoch); + if (epoch > diskState.maxEpoch) + { + diskState = diskState.withNewMaxEpoch(epoch); + saveEpochDiskState(diskState); + } + return diskState; + } + + public static EpochDiskState saveTopology(Topology topology, EpochDiskState diskState) + { + diskState = maybeUpdateMaxEpoch(diskState, topology.epoch()); + + try + { + String cql = "UPDATE %s.%s SET topology=? WHERE epoch=?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), + serialize(topology, LocalVersionedSerializers.topology), topology.epoch()); + flush(Topologies); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + + return diskState; + } + + public static EpochDiskState markRemoteTopologySync(Node.Id node, long epoch, EpochDiskState diskState) + { + diskState = maybeUpdateMaxEpoch(diskState, epoch); + String cql = "UPDATE %s.%s SET remote_sync_complete = remote_sync_complete + ? WHERE epoch = ?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), + Collections.singleton(node.id), epoch); + flush(Topologies); + return diskState; + } + + public static EpochDiskState setNotifyingLocalSync(long epoch, Set pending, EpochDiskState diskState) + { + diskState = maybeUpdateMaxEpoch(diskState, epoch); + String cql = "UPDATE %s.%s SET sync_state = ?, pending_sync_notify = ? WHERE epoch = ?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), + SyncStatus.NOTIFYING.ordinal(), + pending.stream().map(i -> i.id).collect(Collectors.toSet()), + epoch); + return diskState; + } + + public static EpochDiskState markLocalSyncAck(Node.Id node, long epoch, EpochDiskState diskState) + { + diskState = maybeUpdateMaxEpoch(diskState, epoch); + String cql = "UPDATE %s.%s SET pending_sync_notify = pending_sync_notify - ? WHERE epoch = ?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), + Collections.singleton(node.id), epoch); + return diskState; + } + + public static EpochDiskState setCompletedLocalSync(long epoch, EpochDiskState diskState) + { + diskState = maybeUpdateMaxEpoch(diskState, epoch); + String cql = "UPDATE %s.%s SET sync_state = ?, pending_sync_notify = {} WHERE epoch = ?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), + SyncStatus.COMPLETED.ordinal(), + epoch); + return diskState; + } + + public static EpochDiskState truncateTopologyUntil(final long epoch, EpochDiskState diskState) + { + while (diskState.minEpoch < epoch) + { + long delete = diskState.minEpoch; + diskState = diskState.withNewMinEpoch(delete + 1); + saveEpochDiskState(diskState); + String cql = "DELETE FROM %s.%s WHERE epoch = ?"; + executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TOPOLOGIES), delete); + } + return diskState; + } + + public interface TopologyLoadConsumer + { + void load(long epoch, Topology topology, SyncStatus syncStatus, Set pendingSyncNotify, Set remoteSyncComplete); + } + + @VisibleForTesting + public static void loadEpoch(long epoch, TopologyLoadConsumer consumer) throws IOException + { + String cql = format("SELECT * FROM %s.%s WHERE epoch=?", ACCORD_KEYSPACE_NAME, TOPOLOGIES); + + UntypedResultSet result = executeInternal(cql, epoch); + Invariants.checkState(!result.isEmpty(), "Nothing found for epoch %d", epoch); + UntypedResultSet.Row row = result.one(); + Topology topology = row.has("topology") + ? deserialize(row.getBytes("topology"), LocalVersionedSerializers.topology) + : null; + + SyncStatus syncStatus = row.has("sync_state") + ? SyncStatus.values()[row.getInt("sync_state")] + : SyncStatus.NOT_STARTED; + Set pendingSyncNotify = row.has("pending_sync_notify") + ? row.getSet("pending_sync_notify", Int32Type.instance).stream().map(Node.Id::new).collect(Collectors.toSet()) + : Collections.emptySet(); + Set remoteSyncComplete = row.has("remote_sync_complete") + ? row.getSet("remote_sync_complete", Int32Type.instance).stream().map(Node.Id::new).collect(Collectors.toSet()) + : Collections.emptySet(); + + consumer.load(epoch, topology, syncStatus, pendingSyncNotify, remoteSyncComplete); + + } + + public static EpochDiskState loadTopologies(TopologyLoadConsumer consumer) + { + try + { + EpochDiskState diskState = loadEpochDiskState(); + if (diskState == null) + return EpochDiskState.EMPTY; + + for (long epoch=diskState.minEpoch; epoch<=diskState.maxEpoch; epoch++) + loadEpoch(epoch, consumer); + + return diskState; + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + private static IMutation getCommandStoreMetadataMutation(String cql, ByteBuffer... values) + { + ClientState clientState = ClientState.forInternalCalls(); + ModificationStatement statement = (ModificationStatement) QueryProcessor.parseStatement(cql).prepare(ClientState.forInternalCalls()); + QueryOptions options = QueryOptions.forInternalCalls(Arrays.asList(values)); + + long tsMicros = TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); + + while (true) + { + long prev = commandStoreMetadataTimestamp.get(); + if (prev >= tsMicros) + tsMicros = prev + 1; + + if (commandStoreMetadataTimestamp.compareAndSet(prev, tsMicros)) + break; + } + + return Iterables.getOnlyElement(statement.getMutations(clientState, options, true, tsMicros, (int) TimeUnit.MICROSECONDS.toSeconds(tsMicros), Dispatcher.RequestTime.forImmediateExecution())); + } + + + private static Future updateCommandStoreMetadata(CommandStore commandStore, String column, T value, LocalVersionedSerializer serializer) + { + String cql = format("UPDATE %s.%s SET %s=? WHERE store_id=?", ACCORD_KEYSPACE_NAME, COMMAND_STORE_METADATA, column); + try + { + IMutation mutation = getCommandStoreMetadataMutation(cql, serialize(value, serializer), bytes(commandStore.id())); + return Stage.MUTATION.submit(mutation::apply); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + public static Future updateRejectBefore(CommandStore commandStore, ReducingRangeMap rejectBefore) + { + return updateCommandStoreMetadata(commandStore, "reject_before", rejectBefore, LocalVersionedSerializers.rejectBefore); + } + + public static Future updateBootstrapBeganAt(CommandStore commandStore, NavigableMap bootstrapBeganAt) + { + return updateCommandStoreMetadata(commandStore, "bootstrap_began_at", bootstrapBeganAt, LocalVersionedSerializers.bootstrapBeganAt); + } + + public static Future updateSafeToRead(CommandStore commandStore, NavigableMap safeToRead) + { + return updateCommandStoreMetadata(commandStore, "safe_to_read", safeToRead, LocalVersionedSerializers.safeToRead); + } + + public interface CommandStoreMetadataConsumer + { + void accept(ReducingRangeMap rejectBefore, NavigableMap bootstrapBeganAt, NavigableMap safeToRead); + + } + public static void loadCommandStoreMetadata(int id, CommandStoreMetadataConsumer consumer) + { + UntypedResultSet result = executeOnceInternal(format("SELECT * FROM %s.%s WHERE store_id=?", ACCORD_KEYSPACE_NAME, COMMAND_STORE_METADATA), id); + ReducingRangeMap rejectBefore = null; + NavigableMap bootstrapBeganAt = null; + NavigableMap safeToRead = null; + if (!result.isEmpty()) + { + UntypedResultSet.Row row = Iterables.getOnlyElement(result); + try + { + if (row.has("reject_before")) + rejectBefore = deserialize(row.getBlob("reject_before"), LocalVersionedSerializers.rejectBefore); + if (row.has("bootstrap_began_at")) + bootstrapBeganAt = deserialize(row.getBlob("bootstrap_began_at"), LocalVersionedSerializers.bootstrapBeganAt); + if (row.has("safe_to_read")) + safeToRead = deserialize(row.getBlob("safe_to_read"), LocalVersionedSerializers.safeToRead); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + consumer.accept(rejectBefore, bootstrapBeganAt, safeToRead); + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordLocalSyncNotifier.java b/src/java/org/apache/cassandra/service/accord/AccordLocalSyncNotifier.java new file mode 100644 index 0000000000..ade2f8e3e3 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/AccordLocalSyncNotifier.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.local.Node; +import accord.utils.Invariants; +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.accord.serializers.TopologySerializers; + +public class AccordLocalSyncNotifier implements RequestCallback +{ + public static final IVerbHandler verbHandler = message -> AccordService.instance().remoteSyncComplete(message); + private static final Logger logger = LoggerFactory.getLogger(AccordLocalSyncNotifier.class); + + interface Listener + { + void onEndpointAck(Node.Id id, long epoch); + void onComplete(long epoch); + } + + private final long epoch; + private final Node.Id from; + private final Set pendingNotifications; + private final AccordEndpointMapper endpointMapper; + private final IFailureDetector failureDetector; + private final Listener listener; + private final MessageDelivery messagingService; + + public AccordLocalSyncNotifier(long epoch, + Node.Id from, Set pendingNotifications, + AccordEndpointMapper endpointMapper, + MessageDelivery messagingService, IFailureDetector failureDetector, + Listener listener) + { + this.epoch = epoch; + this.from = from; + this.pendingNotifications = pendingNotifications; + this.endpointMapper = endpointMapper; + this.failureDetector = failureDetector; + this.listener = listener; + this.messagingService = messagingService; + } + + private void notify(Node.Id to) + { + InetAddressAndPort toEp = endpointMapper.mappedEndpoint(to); + if (failureDetector.isAlive(toEp)) + { + Message msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, new Notification(epoch, from, to)); + messagingService.sendWithCallback(msg, toEp, this); + } + else + { + scheduleNotify(to); + } + } + + public void scheduleNotify(Node.Id to) + { + ScheduledExecutors.scheduledTasks.schedule(() -> notify(to), 1, TimeUnit.MINUTES); + } + + public synchronized void start() + { + if (pendingNotifications.isEmpty()) + { + listener.onComplete(epoch); + return; + } + pendingNotifications.forEach(this::notify); + } + + private synchronized void onResponse(InetAddressAndPort fromEp, Node.Id from) + { + try + { + Invariants.checkArgument(endpointMapper.mappedId(fromEp).equals(from), "%s != %s", from, endpointMapper.mappedId(fromEp)); + listener.onEndpointAck(from, epoch); + pendingNotifications.remove(from); + if (pendingNotifications.isEmpty()) + listener.onComplete(epoch); + } + catch (Throwable t) + { + logger.error("Unhandled exception handling sync ack on epoch {} from {}", epoch, fromEp, t); + scheduleNotify(from); + } + } + + @Override + public synchronized void onResponse(Message msg) + { + onResponse(msg.from(), msg.payload.from); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + scheduleNotify(endpointMapper.mappedId(from)); + } + + public static class Notification + { + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(Notification notification, DataOutputPlus out, int version) throws IOException + { + out.writeLong(notification.epoch); + TopologySerializers.nodeId.serialize(notification.from, out, version); + TopologySerializers.nodeId.serialize(notification.to, out, version); + } + + @Override + public Notification deserialize(DataInputPlus in, int version) throws IOException + { + return new Notification(in.readLong(), + TopologySerializers.nodeId.deserialize(in, version), + TopologySerializers.nodeId.deserialize(in, version)); + } + + @Override + public long serializedSize(Notification notification, int version) + { + return TypeSizes.LONG_SIZE + + TopologySerializers.nodeId.serializedSize() + + TopologySerializers.nodeId.serializedSize(); + } + }; + final long epoch; + final Node.Id from; + final Node.Id to; + + public Notification(long epoch, Node.Id from, Node.Id to) + { + this.epoch = epoch; + this.from = from; + this.to = to; + } + } + + public static class Acknowledgement + { + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(Acknowledgement acknowledgement, DataOutputPlus out, int version) throws IOException + { + TopologySerializers.nodeId.serialize(acknowledgement.from, out, version); + } + + @Override + public Acknowledgement deserialize(DataInputPlus in, int version) throws IOException + { + return new Acknowledgement(TopologySerializers.nodeId.deserialize(in, version)); + } + + @Override + public long serializedSize(Acknowledgement acknowledgement, int version) + { + return TopologySerializers.nodeId.serializedSize(); + } + }; + + final Node.Id from; + + public Acknowledgement(Node.Id from) + { + this.from = from; + } + } +} diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index 034504d7e7..9169b91936 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -38,8 +38,6 @@ import accord.messages.ReplyContext; import accord.messages.Request; import org.apache.cassandra.locator.InetAddressAndPort; -import static org.apache.cassandra.service.accord.EndpointMapping.getEndpoint; - public class AccordMessageSink implements MessageSink { private static final Logger logger = LoggerFactory.getLogger(AccordMessageSink.class); @@ -77,6 +75,8 @@ public class AccordMessageSink implements MessageSink mapping.put(MessageType.GET_DEPS_REQ, Verb.ACCORD_GET_DEPS_REQ); mapping.put(MessageType.GET_DEPS_RSP, Verb.ACCORD_GET_DEPS_RSP); mapping.put(MessageType.SIMPLE_RSP, Verb.ACCORD_SIMPLE_RSP); + mapping.put(MessageType.FETCH_DATA_REQ, Verb.ACCORD_FETCH_DATA_REQ); + mapping.put(MessageType.FETCH_DATA_RSP, Verb.ACCORD_FETCH_DATA_RSP); for (MessageType type : MessageType.values()) { @@ -93,16 +93,18 @@ public class AccordMessageSink implements MessageSink private final Agent agent; private final MessageDelivery messaging; + private final AccordEndpointMapper endpointMapper; - public AccordMessageSink(Agent agent, MessageDelivery messaging) + public AccordMessageSink(Agent agent, MessageDelivery messaging, AccordEndpointMapper endpointMapper) { this.agent = agent; this.messaging = messaging; + this.endpointMapper = endpointMapper; } - public AccordMessageSink(Agent agent) + public AccordMessageSink(Agent agent, AccordConfigurationService endpointMapper) { - this(agent, MessagingService.instance()); + this(agent, MessagingService.instance(), endpointMapper); } @Override @@ -111,7 +113,7 @@ public class AccordMessageSink implements MessageSink Verb verb = getVerb(request.type()); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); - InetAddressAndPort endpoint = getEndpoint(to); + InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); logger.debug("Sending {} {} to {}", verb, message.payload, endpoint); messaging.send(message, endpoint); } @@ -122,9 +124,9 @@ public class AccordMessageSink implements MessageSink Verb verb = getVerb(request.type()); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); - InetAddressAndPort endpoint = getEndpoint(to); + InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); logger.debug("Sending {} {} to {}", verb, message.payload, endpoint); - messaging.sendWithCallback(message, endpoint, new AccordCallback<>(executor, (Callback) callback)); + messaging.sendWithCallback(message, endpoint, new AccordCallback<>(executor, (Callback) callback, endpointMapper)); } @Override @@ -135,7 +137,7 @@ public class AccordMessageSink implements MessageSink Verb verb = getVerb(reply.type()); Preconditions.checkNotNull(verb, "Verb is null for type %s", reply.type()); Preconditions.checkArgument(replyMsg.verb() == verb, "Expected reply message with verb %s but got %s; reply type was %s", replyMsg.verb(), verb, reply.type()); - InetAddressAndPort endpoint = getEndpoint(replyingToNode); + InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode); logger.debug("Replying {} {} to {}", replyMsg.verb(), replyMsg.payload, endpoint); messaging.send(replyMsg, endpoint); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordService.java b/src/java/org/apache/cassandra/service/accord/AccordService.java index 3a979c1332..09db36757f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordService.java @@ -40,6 +40,8 @@ import accord.primitives.Txn; import accord.primitives.TxnId; import accord.topology.TopologyManager; import accord.utils.DefaultRandom; +import accord.utils.Invariants; +import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import org.apache.cassandra.concurrent.Shutdownable; import accord.utils.async.AsyncResult; @@ -50,15 +52,22 @@ import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.metrics.AccordClientRequestMetrics; import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter; import org.apache.cassandra.service.accord.api.AccordScheduler; import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException; import org.apache.cassandra.service.accord.exceptions.WritePreemptedException; import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner; @@ -70,12 +79,14 @@ public class AccordService implements IAccordService, Shutdownable public static final AccordClientRequestMetrics readMetrics = new AccordClientRequestMetrics("AccordRead"); public static final AccordClientRequestMetrics writeMetrics = new AccordClientRequestMetrics("AccordWrite"); + private static final Future BOOTSTRAP_SUCCESS = ImmediateFuture.success(null); private final Node node; private final Shutdownable nodeShutdown; private final AccordMessageSink messageSink; private final AccordConfigurationService configService; private final AccordScheduler scheduler; + private final AccordDataStore dataStore; private final AccordVerbHandler verbHandler; private static final IAccordService NOOP_SERVICE = new IAccordService() @@ -86,9 +97,6 @@ public class AccordService implements IAccordService, Shutdownable return null; } - @Override - public void createEpochFromConfigUnsafe() { } - @Override public TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel) { @@ -110,15 +118,39 @@ public class AccordService implements IAccordService, Shutdownable throw new UnsupportedOperationException("Cannot return topology when accord_transactions_enabled = false in cassandra.yaml"); } + @Override + public void startup() {} + @Override public void shutdownAndWait(long timeout, TimeUnit unit) { } + + @Override + public Future epochReady(Epoch epoch) + { + return BOOTSTRAP_SUCCESS; + } + + @Override + public void remoteSyncComplete(Message message) {} + + public boolean isAccordManagedKeyspace(String keyspace) + { + return false; + } }; + private static Node.Id localId = null; private static class Handle { public static final AccordService instance = new AccordService(); } + public static void startup(NodeId tcmId) + { + localId = AccordTopologyUtils.tcmIdToAccord(tcmId); + instance().startup(); + } + public static IAccordService instance() { return DatabaseDescriptor.getAccordTransactionsEnabled() ? Handle.instance : NOOP_SERVICE; @@ -131,17 +163,18 @@ public class AccordService implements IAccordService, Shutdownable private AccordService() { - Node.Id localId = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort()); + Invariants.checkState(localId != null, "static localId must be set before instantiating AccordService"); logger.info("Starting accord with nodeId {}", localId); AccordAgent agent = new AccordAgent(); - this.messageSink = new AccordMessageSink(agent); this.configService = new AccordConfigurationService(localId); + this.messageSink = new AccordMessageSink(agent, configService); this.scheduler = new AccordScheduler(); + this.dataStore = new AccordDataStore(); this.node = new Node(localId, messageSink, configService, AccordService::uniqueNow, - () -> AccordDataStore.INSTANCE, + () -> dataStore, new KeyspaceSplitter(new EvenSplit<>(DatabaseDescriptor.getAccordShardCount(), getPartitioner().accordSplitter())), agent, new DefaultRandom(), @@ -150,7 +183,14 @@ public class AccordService implements IAccordService, Shutdownable SimpleProgressLog::new, AccordCommandStores.factory(new AccordJournal().start())); this.nodeShutdown = toShutdownable(node); - this.verbHandler = new AccordVerbHandler<>(this.node); + this.verbHandler = new AccordVerbHandler<>(this.node, configService); + } + + @Override + public void startup() + { + configService.start(); + ClusterMetadataService.instance().log().addListener(configService); } @Override @@ -159,13 +199,6 @@ public class AccordService implements IAccordService, Shutdownable return verbHandler; } - @Override - @VisibleForTesting - public void createEpochFromConfigUnsafe() - { - configService.createEpochFromConfig(); - } - public static long nowInMicros() { return TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis()); @@ -248,12 +281,6 @@ public class AccordService implements IAccordService, Shutdownable : new ReadPreemptedException(consistencyLevel, 0, 0, false, txnId.toString()); } - @VisibleForTesting - AccordMessageSink messageSink() - { - return messageSink; - } - @Override public void setCacheSize(long kb) { @@ -308,6 +335,26 @@ public class AccordService implements IAccordService, Shutdownable return node; } + @Override + public Future epochReady(Epoch epoch) + { + AsyncPromise promise = new AsyncPromise<>(); + AsyncChain ready = configService.epochReady(epoch.getEpoch()); + ready.begin((result, failure) -> { + if (failure == null) promise.trySuccess(result); + else promise.tryFailure(failure); + }); + return promise; + } + + @Override + public void remoteSyncComplete(Message message) + { + Invariants.checkArgument(localId.equals(message.payload.to), "%s != %s", localId, message.payload.to); + configService.remoteSyncComplete(message.payload.from, message.payload.epoch); + MessagingService.instance().respond(new AccordLocalSyncNotifier.Acknowledgement(localId), message); + } + private static Shutdownable toShutdownable(Node node) { return new Shutdownable() { @@ -343,4 +390,15 @@ public class AccordService implements IAccordService, Shutdownable } }; } + + @VisibleForTesting + public AccordConfigurationService configurationService() + { + return configService; + } + + public boolean isAccordManagedKeyspace(String keyspace) + { + return configService.isAccordManagedKeyspace(keyspace); + } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java b/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java index 1b69c27aa4..f234738402 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTopologyUtils.java @@ -19,84 +19,128 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; -import java.util.Comparator; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.common.collect.Sets; + +import accord.local.Node; import accord.topology.Shard; import accord.topology.Topology; -import org.apache.cassandra.db.Keyspace; +import accord.utils.Invariants; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; -import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey; import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.DataPlacements; public class AccordTopologyUtils { - private static Shard createShard(TokenRange range, EndpointsForToken natural, EndpointsForToken pending) + static Node.Id tcmIdToAccord(NodeId nodeId) { - return new Shard(range, - natural.stream().map(EndpointMapping::getId).collect(Collectors.toList()), - natural.stream().map(EndpointMapping::getId).collect(Collectors.toSet()), - pending.stream().map(EndpointMapping::getId).collect(Collectors.toSet())); + return new Node.Id(nodeId.id()); } - private static TokenRange minRange(String keyspace, Token token) + private static Shard createShard(TokenRange range, Directory directory, EndpointsForRange reads, EndpointsForRange writes) + { + Function endpointMapper = e -> { + NodeId tcmId = directory.peerId(e); + return tcmIdToAccord(tcmId); + }; + Set endpoints = reads.endpoints(); + Set writeEndpoints = writes.endpoints(); + List nodes = endpoints.stream().map(endpointMapper).sorted().collect(Collectors.toList()); + Set fastPath = new HashSet<>(nodes); // TODO: support fast path updates + Set pending = endpoints.equals(writeEndpoints) ? + Collections.emptySet() : + writeEndpoints.stream().filter(e -> !endpoints.contains(e)).map(endpointMapper).collect(Collectors.toSet()); + + Sets.SetView readOnly = Sets.difference(endpoints, writeEndpoints); + Invariants.checkState(readOnly.isEmpty(), "Read only replicas detected: %s", readOnly); + return new Shard(range, nodes, fastPath, pending); + } + + static TokenRange minRange(String keyspace, Token token) { return new TokenRange(SentinelKey.min(keyspace), new TokenKey(keyspace, token)); } - private static TokenRange maxRange(String keyspace, Token token) + static TokenRange maxRange(String keyspace, Token token) { return new TokenRange(new TokenKey(keyspace, token), SentinelKey.max(keyspace)); } - private static TokenRange range(String keyspace, Token left, Token right) + static TokenRange fullRange(String keyspace) { - return new TokenRange(new TokenKey(keyspace, left), new TokenKey(keyspace, right)); + return new TokenRange(SentinelKey.min(keyspace), SentinelKey.max(keyspace)); } - public static List createShards(String keyspace, ClusterMetadata clusterMetadata) + static TokenRange range(String keyspace, Range range) { - KeyspaceMetadata keyspaceMetadata = Keyspace.open(keyspace).getMetadata(); - List tokens = new ArrayList<>(clusterMetadata.tokenMap.tokens()); - tokens.sort(Comparator.naturalOrder()); + Token minToken = range.left.minValue(); + return new TokenRange(range.left.equals(minToken) ? SentinelKey.min(keyspace) : new TokenKey(keyspace, range.left), + range.right.equals(minToken) ? SentinelKey.max(keyspace) : new TokenKey(keyspace, range.right)); + } - List shards = new ArrayList<>(tokens.size() + 1); - Shard finalShard = null; - for (int i = 0, mi = tokens.size(); i < mi; i++) + public static List createShards(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory) + { + ReplicationParams replication = keyspace.params.replication; + DataPlacement placement = placements.get(replication); + + List> ranges = placement.reads.ranges(); + List shards = new ArrayList<>(ranges.size()); + for (Range range : ranges) { - Token token = tokens.get(i); - EndpointsForToken natural = clusterMetadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get(); - EndpointsForToken pending = clusterMetadata.pendingEndpointsFor(keyspaceMetadata, token).get(); - if (i == 0) - { - shards.add(createShard(minRange(keyspace, token), natural, pending)); - finalShard = createShard(maxRange(keyspace, tokens.get(mi - 1)), natural, pending); - } - else - { - Token prev = tokens.get(i - 1); - shards.add(createShard(range(keyspace, prev, token), natural, pending)); - } + EndpointsForRange reads = placement.reads.forRange(range).get(); + EndpointsForRange writes = placement.reads.forRange(range).get(); + + // TCM doesn't create wrap around ranges + Invariants.checkArgument(!range.isWrapAround() || range.right.equals(range.right.minValue()), + "wrap around range %s found", range); + shards.add(createShard(range(keyspace.name, range), directory, reads, writes)); } - shards.add(finalShard); return shards; } - public static Topology createTopology(long epoch) + public static Topology createAccordTopology(Epoch epoch, DistributedSchema schema, DataPlacements placements, Directory directory, Predicate keyspacePredicate) { - List keyspaces = new ArrayList<>(Schema.instance.distributedKeyspaces().names()); - keyspaces.sort(String::compareTo); - List shards = new ArrayList<>(); - for (String keyspace : keyspaces) - shards.addAll(createShards(keyspace, ClusterMetadata.current())); + for (KeyspaceMetadata keyspace : schema.getKeyspaces()) + { + if (!keyspacePredicate.test(keyspace.name)) + continue; + shards.addAll(createShards(keyspace, placements, directory)); + } + shards.sort((a, b) -> a.range.compare(b.range)); + return new Topology(epoch.getEpoch(), shards.toArray(new Shard[0])); + } - return new Topology(epoch, shards.toArray(new Shard[0])); + public static EndpointMapping directoryToMapping(long epoch, Directory directory) + { + EndpointMapping.Builder builder = EndpointMapping.builder(epoch); + for (NodeId id : directory.peerIds()) + builder.add(directory.endpoint(id), tcmIdToAccord(id)); + return builder.build(); + } + + public static Topology createAccordTopology(ClusterMetadata metadata, Predicate keyspacePredicate) + { + return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, keyspacePredicate); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java b/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java index a35737040d..5cf7e95aaa 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java +++ b/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java @@ -33,18 +33,21 @@ public class AccordVerbHandler implements IVerbHandler private static final Logger logger = LoggerFactory.getLogger(AccordVerbHandler.class); private final Node node; + private final AccordEndpointMapper endpointMapper; - public AccordVerbHandler(Node node) + public AccordVerbHandler(Node node, AccordEndpointMapper endpointMapper) { this.node = node; + this.endpointMapper = endpointMapper; } @Override public void doVerb(Message message) throws IOException { + // TODO (desired): need a non-blocking way to inform CMS of an unknown epoch and add callback to it's receipt +// ClusterMetadataService.instance().maybeCatchup(message.epoch()); logger.debug("Receiving {} from {}", message.payload, message.from()); T request = message.payload; - Node.Id from = EndpointMapping.getId(message.from()); long knownEpoch = request.knownEpoch(); if (!node.topology().hasEpoch(knownEpoch)) { @@ -52,10 +55,10 @@ public class AccordVerbHandler implements IVerbHandler long waitForEpoch = request.waitForEpoch(); if (!node.topology().hasEpoch(waitForEpoch)) { - node.withEpoch(waitForEpoch, () -> request.process(node, from, message)); + node.withEpoch(waitForEpoch, () -> request.process(node, endpointMapper.mappedId(message.from()), message)); return; } } - request.process(node, from, message); + request.process(node, endpointMapper.mappedId(message.from()), message); } } diff --git a/src/java/org/apache/cassandra/service/accord/EndpointMapping.java b/src/java/org/apache/cassandra/service/accord/EndpointMapping.java index a863ca7410..4a746dc9a2 100644 --- a/src/java/org/apache/cassandra/service/accord/EndpointMapping.java +++ b/src/java/org/apache/cassandra/service/accord/EndpointMapping.java @@ -18,119 +18,70 @@ package org.apache.cassandra.service.accord; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.net.UnknownHostException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableMap; -import com.google.common.primitives.Ints; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableBiMap; import accord.local.Node; +import accord.utils.Invariants; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -public class EndpointMapping +class EndpointMapping implements AccordEndpointMapper { - static Node.Id endpointToId(InetAddressAndPort endpoint) + public static final EndpointMapping EMPTY = new EndpointMapping(0, ImmutableBiMap.of()); + private final long epoch; + private final ImmutableBiMap mapping; + + private EndpointMapping(long epoch, + ImmutableBiMap mapping) { - Preconditions.checkArgument(endpoint.getAddress() instanceof Inet4Address); - Inet4Address address = (Inet4Address) endpoint.getAddress(); - int id = Ints.fromByteArray(address.getAddress()); - return new Node.Id(id); + this.epoch = epoch; + this.mapping = mapping; } - static InetAddressAndPort idToEndpoint(Node.Id node) + long epoch() { - byte[] bytes = Ints.toByteArray(node.id); - try + return epoch; + } + + @Override + public Node.Id mappedId(InetAddressAndPort endpoint) + { + return mapping.inverse().get(endpoint); + } + + @Override + public InetAddressAndPort mappedEndpoint(Node.Id id) + { + return mapping.get(id); + } + + static class Builder + { + private final long epoch; + private final BiMap mapping = HashBiMap.create(); + + public Builder(long epoch) { - return InetAddressAndPort.getByAddress(InetAddress.getByAddress(bytes)); + this.epoch = epoch; } - catch (UnknownHostException e) + + public Builder add(InetAddressAndPort endpoint, Node.Id id) { - throw new RuntimeException(e); + Invariants.checkArgument(!mapping.containsKey(id), "Mapping already exists for Node.Id %s", id); + Invariants.checkArgument(!mapping.containsValue(endpoint), "Mapping already exists for %s", endpoint); + mapping.put(id, endpoint); + return this; + } + + public EndpointMapping build() + { + return new EndpointMapping(epoch, ImmutableBiMap.copyOf(mapping)); } } - // TODO: Remove this if its one usage in AccordConfigurationService is removed. - public static ImmutableCollection knownIds() + static Builder builder(long epoch) { - return mapping.endpointToId.values(); - } - - private static class Mapping - { - private static final Mapping EMPTY = new Mapping(ImmutableMap.of(), ImmutableMap.of()); - final ImmutableMap idToEndpoint; - final ImmutableMap endpointToId; - - public Mapping(ImmutableMap idToEndpoint, - ImmutableMap endpointToId) - { - this.idToEndpoint = idToEndpoint; - this.endpointToId = endpointToId; - } - - private static ImmutableMap put(ImmutableMap current, K key, V val) - { - return ImmutableMap.builderWithExpectedSize(current.size() + 1).putAll(current).put(key, val).build(); - } - - public Mapping add(InetAddressAndPort endpoint) - { - if (endpointToId.containsKey(endpoint)) - return this; - Node.Id id = endpointToId(endpoint); - return new Mapping(put(idToEndpoint, id, endpoint), put(endpointToId, endpoint, id)); - } - - public Mapping add(Node.Id id) - { - if (idToEndpoint.containsKey(id)) - return this; - - InetAddressAndPort endpoint = idToEndpoint(id); - return new Mapping(put(idToEndpoint, id, endpoint), put(endpointToId, endpoint, id)); - } - } - - private static volatile Mapping mapping = Mapping.EMPTY; - - private EndpointMapping() {} - - public static Node.Id getId(InetAddressAndPort endpoint) - { - Node.Id id = mapping.endpointToId.get(endpoint); - if (id == null) - { - synchronized (EndpointMapping.class) - { - mapping = mapping.add(endpoint); - id = mapping.endpointToId.get(endpoint); - } - } - return id; - } - - // FIXME: put this stuff into the configuration service, where it will eventually live - public static Node.Id getId(Replica replica) - { - return getId(replica.endpoint()); - } - - public static InetAddressAndPort getEndpoint(Node.Id id) - { - InetAddressAndPort endpoint = mapping.idToEndpoint.get(id); - if (endpoint == null) - { - synchronized (EndpointMapping.class) - { - mapping = mapping.add(id); - endpoint = mapping.idToEndpoint.get(id); - } - } - return endpoint; + return new Builder(epoch); } } diff --git a/src/java/org/apache/cassandra/service/accord/IAccordService.java b/src/java/org/apache/cassandra/service/accord/IAccordService.java index 7a01fc52ab..8ba98e919d 100644 --- a/src/java/org/apache/cassandra/service/accord/IAccordService.java +++ b/src/java/org/apache/cassandra/service/accord/IAccordService.java @@ -23,7 +23,10 @@ import accord.primitives.Txn; import accord.topology.TopologyManager; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; import org.apache.cassandra.service.accord.txn.TxnData; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -32,8 +35,6 @@ public interface IAccordService { IVerbHandler verbHandler(); - void createEpochFromConfigUnsafe(); - TxnData coordinate(Txn txn, ConsistencyLevel consistencyLevel); long currentEpoch(); @@ -42,5 +43,22 @@ public interface IAccordService TopologyManager topology(); + void startup(); + void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException; + + /** + * Return a future that will complete once the accord has completed it's local bootstrap process + * for any ranges gained in the given epoch + */ + Future epochReady(Epoch epoch); + + void remoteSyncComplete(Message message); + + /** + * Temporary method to avoid double-streaming keyspaces + * @param keyspace + * @return + */ + boolean isAccordManagedKeyspace(String keyspace); } diff --git a/src/java/org/apache/cassandra/service/accord/TokenRange.java b/src/java/org/apache/cassandra/service/accord/TokenRange.java index a0b6f67ccc..3c8e0e76ea 100644 --- a/src/java/org/apache/cassandra/service/accord/TokenRange.java +++ b/src/java/org/apache/cassandra/service/accord/TokenRange.java @@ -23,6 +23,10 @@ import java.io.IOException; import accord.api.RoutingKey; import accord.primitives.Range; import accord.primitives.Ranges; +import accord.utils.Invariants; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -34,6 +38,14 @@ public class TokenRange extends Range.EndInclusive public TokenRange(AccordRoutingKey start, AccordRoutingKey end) { super(start, end); + Invariants.checkArgument(start.keyspace().equals(end.keyspace()), + "Token ranges cannot cover more than one keyspace start:%s, end:%s", + start, end); + } + + public String keyspace() + { + return ((AccordRoutingKey) start()).keyspace(); } public static TokenRange fullRange(String keyspace) @@ -56,6 +68,16 @@ public class TokenRange extends Range.EndInclusive return pick; } + public org.apache.cassandra.dht.Range toKeyspaceRange () + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + AccordRoutingKey start = (AccordRoutingKey) start(); + AccordRoutingKey end = (AccordRoutingKey) end(); + Token left = start instanceof SentinelKey ? partitioner.getMinimumToken() : start.token(); + Token right = end instanceof SentinelKey ? partitioner.getMinimumToken() : end.token(); + return new org.apache.cassandra.dht.Range<>(left, right); + } + public static final IVersionedSerializer serializer = new IVersionedSerializer() { @Override diff --git a/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java b/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java index 9a1ca51c6f..b5eeaa3f38 100644 --- a/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java +++ b/src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java @@ -170,8 +170,7 @@ public abstract class AsyncOperation extends AsyncChains.Head implements R { Invariants.nonNull(throwable); if (state.isComplete()) - throw new IllegalStateException("Unexpected state " + state, throwable); - + return; try { switch (state) diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java index 366b469c2f..4b20e9e7ea 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java @@ -22,6 +22,9 @@ import java.io.IOException; import com.google.common.base.Preconditions; +import accord.api.Query; +import accord.api.Read; +import accord.api.Update; import accord.local.Node; import accord.local.SaveStatus; import accord.local.Status; @@ -43,6 +46,7 @@ import org.apache.cassandra.service.accord.txn.TxnQuery; import org.apache.cassandra.service.accord.txn.TxnRead; import org.apache.cassandra.service.accord.txn.TxnUpdate; import org.apache.cassandra.service.accord.txn.TxnWrite; +import org.apache.cassandra.utils.CastingSerializer; public class CommandSerializers { @@ -118,19 +122,30 @@ public class CommandSerializers } } - public static final IVersionedSerializer partialTxn = new IVersionedSerializer() + public static class PartialTxnSerializer implements IVersionedSerializer { + private final IVersionedSerializer readSerializer; + private final IVersionedSerializer querySerializer; + private final IVersionedSerializer updateSerializer; + + public PartialTxnSerializer(IVersionedSerializer readSerializer, IVersionedSerializer querySerializer, IVersionedSerializer updateSerializer) + { + this.readSerializer = readSerializer; + this.querySerializer = querySerializer; + this.updateSerializer = updateSerializer; + } + @Override public void serialize(PartialTxn txn, DataOutputPlus out, int version) throws IOException { CommandSerializers.kind.serialize(txn.kind(), out, version); KeySerializers.ranges.serialize(txn.covering(), out, version); KeySerializers.seekables.serialize(txn.keys(), out, version); - TxnRead.serializer.serialize((TxnRead) txn.read(), out, version); - TxnQuery.serializer.serialize((TxnQuery) txn.query(), out, version); + readSerializer.serialize(txn.read(), out, version); + querySerializer.serialize(txn.query(), out, version); out.writeBoolean(txn.update() != null); if (txn.update() != null) - TxnUpdate.serializer.serialize((TxnUpdate) txn.update(), out, version); + updateSerializer.serialize(txn.update(), out, version); } @Override @@ -139,9 +154,9 @@ public class CommandSerializers Txn.Kind kind = CommandSerializers.kind.deserialize(in, version); Ranges covering = KeySerializers.ranges.deserialize(in, version); Seekables keys = KeySerializers.seekables.deserialize(in, version); - TxnRead read = TxnRead.serializer.deserialize(in, version); - TxnQuery query = TxnQuery.serializer.deserialize(in, version); - TxnUpdate update = in.readBoolean() ? TxnUpdate.serializer.deserialize(in, version) : null; + Read read = readSerializer.deserialize(in, version); + Query query = querySerializer.deserialize(in, version); + Update update = in.readBoolean() ? updateSerializer.deserialize(in, version) : null; return new PartialTxn.InMemory(covering, kind, keys, read, query, update); } @@ -151,14 +166,20 @@ public class CommandSerializers long size = CommandSerializers.kind.serializedSize(txn.kind(), version); size += KeySerializers.ranges.serializedSize(txn.covering(), version); size += KeySerializers.seekables.serializedSize(txn.keys(), version); - size += TxnRead.serializer.serializedSize((TxnRead) txn.read(), version); - size += TxnQuery.serializer.serializedSize((TxnQuery) txn.query(), version); + size += readSerializer.serializedSize(txn.read(), version); + size += querySerializer.serializedSize(txn.query(), version); size += TypeSizes.sizeof(txn.update() != null); if (txn.update() != null) - size += TxnUpdate.serializer.serializedSize((TxnUpdate) txn.update(), version); + size += updateSerializer.serializedSize(txn.update(), version); return size; } - }; + } + + private static final IVersionedSerializer read = new CastingSerializer<>(TxnRead.class, TxnRead.serializer); + private static final IVersionedSerializer query = new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer); + private static final IVersionedSerializer update = new CastingSerializer<>(TxnUpdate.class, TxnUpdate.serializer); + + public static final IVersionedSerializer partialTxn = new PartialTxnSerializer(read, query, update); public static final IVersionedSerializer saveStatus = new EnumSerializer<>(SaveStatus.class); public static final IVersionedSerializer status = new EnumSerializer<>(Status.class); diff --git a/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java new file mode 100644 index 0000000000..4cd322435c --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/serializers/CommandStoreSerializers.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.serializers; + +import java.io.IOException; +import java.util.NavigableMap; +import java.util.TreeMap; + +import accord.api.RoutingKey; +import accord.primitives.Ranges; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.utils.ReducingRangeMap; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.CollectionSerializers; + +public class CommandStoreSerializers +{ + private CommandStoreSerializers() {} + + public static IVersionedSerializer> rejectBefore = new IVersionedSerializer>() + { + public void serialize(ReducingRangeMap map, DataOutputPlus out, int version) throws IOException + { + out.writeBoolean(map.inclusiveEnds()); + int size = map.size(); + out.writeUnsignedVInt32(size); + + for (int i=0; i deserialize(DataInputPlus in, int version) throws IOException + { + boolean inclusiveEnds = in.readBoolean(); + int size = in.readUnsignedVInt32(); + RoutingKey[] keys = new RoutingKey[size]; + Timestamp[] values = new Timestamp[size + 1]; + for (int i=0; i map, int version) + { + long size = TypeSizes.BOOL_SIZE; + size += TypeSizes.sizeofUnsignedVInt(size); + int mapSize = map.size(); + for (int i=0; i implements IVersionedSerializer> + { + private final IVersionedSerializer timestampSerializer; + + public TimestampToRangesSerializer(IVersionedSerializer timestampSerializer) + { + this.timestampSerializer = timestampSerializer; + } + + public void serialize(NavigableMap map, DataOutputPlus out, int version) throws IOException + { + CollectionSerializers.serializeMap(map, out, version, timestampSerializer, KeySerializers.ranges); + } + + public NavigableMap deserialize(DataInputPlus in, int version) throws IOException + { + return CollectionSerializers.deserializeMap(in, version, timestampSerializer, KeySerializers.ranges, i -> new TreeMap<>()); + + } + + public long serializedSize(NavigableMap map, int version) + { + return CollectionSerializers.serializedMapSize(map, version, timestampSerializer, KeySerializers.ranges); + } + } + + public static final IVersionedSerializer> bootstrapBeganAt = new TimestampToRangesSerializer<>(CommandSerializers.txnId); + public static final IVersionedSerializer> safeToRead = new TimestampToRangesSerializer<>(CommandSerializers.timestamp); +} diff --git a/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java new file mode 100644 index 0000000000..e76df16335 --- /dev/null +++ b/src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord.serializers; + +import java.io.IOException; + +import accord.api.Data; +import accord.impl.AbstractFetchCoordinator.FetchRequest; +import accord.impl.AbstractFetchCoordinator.FetchResponse; +import accord.local.Status; +import accord.messages.ReadData; +import accord.messages.ReadData.ReadReply; +import accord.primitives.Ranges; +import accord.primitives.Timestamp; +import accord.primitives.TxnId; +import accord.utils.Invariants; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.accord.AccordFetchCoordinator.StreamData; +import org.apache.cassandra.service.accord.AccordFetchCoordinator.StreamingTxn; +import org.apache.cassandra.utils.CastingSerializer; + +import static org.apache.cassandra.utils.NullableSerializer.deserializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializeNullable; +import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSize; + +public class FetchSerializers +{ + public static final IVersionedSerializer request = new IVersionedSerializer() + { + @Override + public void serialize(FetchRequest request, DataOutputPlus out, int version) throws IOException + { + Invariants.checkArgument(request.txnId.equals(TxnId.NONE)); + Invariants.checkArgument(request.waitForStatus == Status.Applied); + Invariants.checkArgument(request.waitUntil.equals(Timestamp.MAX)); + + out.writeUnsignedVInt(request.waitForEpoch()); + CommandSerializers.txnId.serialize((TxnId) request.executeReadAt, out, version); + KeySerializers.ranges.serialize((Ranges) request.readScope, out, version); + + DepsSerializer.partialDeps.serialize(request.partialDeps, out, version); + StreamingTxn.serializer.serialize(request.read, out, version); + } + + @Override + public FetchRequest deserialize(DataInputPlus in, int version) throws IOException + { + return new FetchRequest(in.readUnsignedVInt(), + CommandSerializers.txnId.deserialize(in, version), + KeySerializers.ranges.deserialize(in, version), + DepsSerializer.partialDeps.deserialize(in, version), + StreamingTxn.serializer.deserialize(in, version)); + } + + @Override + public long serializedSize(FetchRequest request, int version) + { + return TypeSizes.sizeofUnsignedVInt(request.waitForEpoch()) + + CommandSerializers.txnId.serializedSize((TxnId) request.executeReadAt, version) + + KeySerializers.ranges.serializedSize((Ranges) request.readScope, version) + + DepsSerializer.partialDeps.serializedSize(request.partialDeps, version) + + StreamingTxn.serializer.serializedSize(request.read, version); + } + }; + + public static final IVersionedSerializer reply = new IVersionedSerializer() + { + final ReadData.ReadNack[] nacks = ReadData.ReadNack.values(); + final IVersionedSerializer streamDataSerializer = new CastingSerializer<>(StreamData.class, StreamData.serializer); + + @Override + public void serialize(ReadReply reply, DataOutputPlus out, int version) throws IOException + { + if (!reply.isOk()) + { + out.writeByte(1 + ((ReadData.ReadNack) reply).ordinal()); + return; + } + + out.writeByte(0); + FetchResponse response = (FetchResponse) reply; + serializeNullable(response.unavailable, out, version, KeySerializers.ranges); + serializeNullable(response.data, out, version, streamDataSerializer); + CommandSerializers.timestamp.serialize(response.maxApplied, out, version); + } + + @Override + public ReadReply deserialize(DataInputPlus in, int version) throws IOException + { + int id = in.readByte(); + if (id != 0) + return nacks[id - 1]; + + return new FetchResponse(deserializeNullable(in, version, KeySerializers.ranges), + deserializeNullable(in, version, streamDataSerializer), + CommandSerializers.timestamp.deserialize(in, version)); + } + + @Override + public long serializedSize(ReadReply reply, int version) + { + if (!reply.isOk()) + return TypeSizes.BYTE_SIZE; + + FetchResponse response = (FetchResponse) reply; + return TypeSizes.BYTE_SIZE + + serializedNullableSize(response.unavailable, version, KeySerializers.ranges) + + serializedNullableSize(response.data, version, streamDataSerializer) + + CommandSerializers.timestamp.serializedSize(response.maxApplied, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java index 04afa5b250..e1f4b5b280 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java @@ -19,13 +19,21 @@ package org.apache.cassandra.service.accord.serializers; import java.io.IOException; +import java.util.List; +import java.util.Set; import accord.local.Node; +import accord.primitives.Range; +import accord.topology.Shard; +import accord.topology.Topology; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.accord.TokenRange; +import org.apache.cassandra.utils.ArraySerializers; +import org.apache.cassandra.utils.CollectionSerializers; public class TopologySerializers { @@ -69,4 +77,64 @@ public class TopologySerializers return TypeSizes.INT_SIZE; // id.id } }; + + public static final IVersionedSerializer shard = new IVersionedSerializer() + { + @Override + public void serialize(Shard shard, DataOutputPlus out, int version) throws IOException + { + TokenRange.serializer.serialize((TokenRange) shard.range, out, version); + CollectionSerializers.serializeList(shard.nodes, out, version, nodeId); + CollectionSerializers.serializeCollection(shard.fastPathElectorate, out, version, nodeId); + CollectionSerializers.serializeCollection(shard.joining, out, version, nodeId); + + } + + @Override + public Shard deserialize(DataInputPlus in, int version) throws IOException + { + Range range = TokenRange.serializer.deserialize(in, version); + List nodes = CollectionSerializers.deserializeList(in, version, nodeId); + Set fastPathElectorate = CollectionSerializers.deserializeSet(in, version, nodeId); + Set joining = CollectionSerializers.deserializeSet(in, version, nodeId); + return new Shard(range, nodes, fastPathElectorate, joining); + } + + @Override + public long serializedSize(Shard shard, int version) + { + long size = TokenRange.serializer.serializedSize((TokenRange) shard.range, version); + size += CollectionSerializers.serializedListSize(shard.nodes, version, nodeId); + size += CollectionSerializers.serializedCollectionSize(shard.fastPathElectorate, version, nodeId); + size += CollectionSerializers.serializedCollectionSize(shard.joining, version, nodeId); + return size; + } + }; + + public static final IVersionedSerializer topology = new IVersionedSerializer() + { + @Override + public void serialize(Topology topology, DataOutputPlus out, int version) throws IOException + { + out.writeLong(topology.epoch()); + ArraySerializers.serializeArray(topology.unsafeGetShards(), out, version, shard); + } + + @Override + public Topology deserialize(DataInputPlus in, int version) throws IOException + { + long epoch = in.readLong(); + Shard[] shards = ArraySerializers.deserializeArray(in, version, shard, Shard[]::new); + return new Topology(epoch, shards); + } + + @Override + public long serializedSize(Topology topology, int version) + { + long size = 0; + size += TypeSizes.LONG_SIZE; // epoch + size += ArraySerializers.serializedArraySize(topology.unsafeGetShards(), version, shard); + return size; + } + }; } diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index 607978806e..e0f1a1c2e1 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -434,6 +434,12 @@ public class ClusterMetadata return this; } + public Transformer register(NodeId nodeId, NodeAddresses addresses, Location location, NodeVersion version) + { + directory = directory.with(nodeId, addresses, location, version); + return this; + } + public Transformer withNodeState(NodeId id, NodeState state) { directory = directory.withNodeState(id, state); diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 6d3fe6b2af..7acfe643c6 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -53,6 +53,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; @@ -415,6 +416,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; { ClusterMetadata metadata = ClusterMetadata.current(); NodeId self = metadata.myNodeId(); + AccordService.startup(self); // finish in-progress sequences first InProgressSequences.finishInProgressSequences(self, true); diff --git a/src/java/org/apache/cassandra/tcm/membership/Directory.java b/src/java/org/apache/cassandra/tcm/membership/Directory.java index 436ded0ab3..a4dab0bc21 100644 --- a/src/java/org/apache/cassandra/tcm/membership/Directory.java +++ b/src/java/org/apache/cassandra/tcm/membership/Directory.java @@ -168,6 +168,11 @@ public class Directory implements MetadataValue return with(addresses, location, CURRENT); } + public Directory with(NodeId id, NodeAddresses addresses, Location location, NodeVersion nodeVersion) + { + return with(addresses, id, id.toUUID(), location, nodeVersion); + } + public Directory with(NodeAddresses addresses, Location location, NodeVersion nodeVersion) { NodeId id = new NodeId(nextId); diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 15182fc92b..607ee2f669 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -25,6 +25,7 @@ import java.util.Set; import java.util.stream.StreamSupport; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +42,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -60,6 +62,7 @@ import org.apache.cassandra.tcm.transformations.PrepareJoin; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.vint.VIntCoding; import static com.google.common.collect.ImmutableList.of; @@ -356,12 +359,14 @@ public class BootstrapAndJoin extends MultiStepOperation StorageService.instance.repairPaxosForTopologyChange("bootstrap"); Future bootstrapStream = StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements); + Future accordReady = AccordService.instance().epochReady(metadata.epoch); + Future ready = FutureCombiner.allOf(Lists.newArrayList(bootstrapStream, accordReady)); try { if (bootstrapTimeoutMillis > 0) - bootstrapStream.get(bootstrapTimeoutMillis, MILLISECONDS); + ready.get(bootstrapTimeoutMillis, MILLISECONDS); else - bootstrapStream.get(); + ready.get(); StorageService.instance.markViewsAsBuilt(); StorageService.instance.clearOngoingBootstrap(); logger.info("Bootstrap completed for tokens {}", tokens); diff --git a/src/java/org/apache/cassandra/tcm/sequences/Move.java b/src/java/org/apache/cassandra/tcm/sequences/Move.java index 99ada06f93..8e83f2e7e3 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/Move.java +++ b/src/java/org/apache/cassandra/tcm/sequences/Move.java @@ -49,8 +49,10 @@ import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; @@ -68,6 +70,8 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.PrepareMove; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.vint.VIntCoding; import static com.google.common.collect.ImmutableList.of; @@ -211,11 +215,12 @@ public class Move extends MultiStepOperation case MID_MOVE: try { + ClusterMetadata metadata = ClusterMetadata.current(); logger.info("fetching new ranges and streaming old ranges"); StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION); Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces(); Map movementMap = movementMap(FailureDetector.instance, - ClusterMetadata.current().placements, + metadata.placements, toSplitRanges, startMove.delta(), midMove.delta(), @@ -224,6 +229,8 @@ public class Move extends MultiStepOperation for (KeyspaceMetadata ks : keyspaces) { + if (AccordService.instance().isAccordManagedKeyspace(ks.name)) + continue; ReplicationParams replicationParams = ks.params.replication; if (replicationParams.isMeta()) continue; @@ -248,7 +255,9 @@ public class Move extends MultiStepOperation } } - streamPlan.execute().get(); + StreamResultFuture streamResult = streamPlan.execute(); + Future accordReady = AccordService.instance().epochReady(metadata.epoch); + FutureCombiner.allOf(streamResult, accordReady).get(); StorageService.instance.repairPaxosForTopologyChange("move"); } catch (InterruptedException e) diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index f1535adcdc..5b149acd9a 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -105,7 +105,6 @@ public class NodeTool Compact.class, CompactionHistory.class, CompactionStats.class, - CreateEpochUnsafe.class, DataPaths.class, Decommission.class, Decommission.Abort.class, diff --git a/src/java/org/apache/cassandra/utils/CastingSerializer.java b/src/java/org/apache/cassandra/utils/CastingSerializer.java new file mode 100644 index 0000000000..6dccb40bbe --- /dev/null +++ b/src/java/org/apache/cassandra/utils/CastingSerializer.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.io.IOException; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * Utility for serializing/deserializing from/into generic interface fields where we know (and require) the + * generic fields to be implementation specific classes + * @param + * @param + */ +public class CastingSerializer implements IVersionedSerializer +{ + private final Class specificClass; + private final IVersionedSerializer specificSerializer; + + public CastingSerializer(Class specificClass, IVersionedSerializer specificSerializer) + { + this.specificClass = specificClass; + this.specificSerializer = specificSerializer; + } + + @Override + public void serialize(Generic generic, DataOutputPlus out, int version) throws IOException + { + specificSerializer.serialize(specificClass.cast(generic), out, version); + } + + @Override + public Generic deserialize(DataInputPlus in, int version) throws IOException + { + Generic result = specificSerializer.deserialize(in, version); + if (result != null && !specificClass.isInstance(result)) + throw new IllegalStateException("Expected instance of " + specificClass.getName()); + return result; + } + + @Override + public long serializedSize(Generic generic, int version) + { + return specificSerializer.serializedSize(specificClass.cast(generic), version); + } +} diff --git a/src/java/org/apache/cassandra/utils/CollectionSerializers.java b/src/java/org/apache/cassandra/utils/CollectionSerializers.java index 93fa02a04d..7e672c431f 100644 --- a/src/java/org/apache/cassandra/utils/CollectionSerializers.java +++ b/src/java/org/apache/cassandra/utils/CollectionSerializers.java @@ -75,10 +75,10 @@ public class CollectionSerializers return deserializeCollection(in, version, serializer, newHashSet()); } - public static Map deserializeMap(DataInputPlus in, int version, IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, IntFunction> factory) throws IOException + public static > M deserializeMap(DataInputPlus in, int version, IVersionedSerializer keySerializer, IVersionedSerializer valueSerializer, IntFunction factory) throws IOException { int size = checkedCast(in.readUnsignedVInt32()); - Map result = factory.apply(size); + M result = factory.apply(size); while (size-- > 0) { K key = keySerializer.deserialize(in, version); diff --git a/src/java/org/apache/cassandra/utils/concurrent/FutureCombiner.java b/src/java/org/apache/cassandra/utils/concurrent/FutureCombiner.java index 325b4baca1..709e5da608 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/FutureCombiner.java +++ b/src/java/org/apache/cassandra/utils/concurrent/FutureCombiner.java @@ -20,6 +20,7 @@ package org.apache.cassandra.utils.concurrent; import io.netty.util.concurrent.GenericFutureListener; import io.netty.util.concurrent.GlobalEventExecutor; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -242,6 +243,11 @@ public class FutureCombiner extends AsyncFuture return new FutureCombiner<>(futures, () -> futures.stream().map(f -> f.getNow()).collect(Collectors.toList()), FailFastListener::new); } + public static Future> allOf(io.netty.util.concurrent.Future... futures) + { + return allOf(Arrays.asList(futures)); + } + /** * Waits for all futures to complete, returning a list containing values of all successful input futures. This * emulates Guava's Futures::successfulAsList in that results will be in the same order as inputs and any diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index a3918443e4..fcbddc2596 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -821,6 +821,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance ClusterMetadataService.instance().processor().fetchLogAndWait(); NodeId self = Register.maybeRegister(); RegistrationStatus.instance.onRegistration(); + AccordService.startup(self); boolean joinRing = config.get(Constants.KEY_DTEST_JOIN_RING) == null || (boolean) config.get(Constants.KEY_DTEST_JOIN_RING); if (ClusterMetadata.current().directory.peerState(self) != NodeState.JOINED && joinRing) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java new file mode 100644 index 0000000000..97071c6678 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.junit.Assert; +import org.junit.Test; + +import accord.local.CommandStore; +import accord.local.PreLoadContext; +import accord.primitives.Timestamp; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordConfigurationService; +import org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.streaming.StreamResultFuture; +import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.Assertions; + +import static accord.utils.async.AsyncChains.awaitUninterruptiblyAndRethrow; +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; + +public class AccordBootstrapTest extends TestBaseImpl +{ + private static DecoratedKey dk(int key) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + return partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + private static PartitionKey pk(int key, String keyspace, String table) + { + TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; + return new PartitionKey(keyspace, tid, dk(key)); + } + + protected void bootstrapAndJoinNode(Cluster cluster) + { + IInstanceConfig config = cluster.newInstanceConfig(); + config.set("auto_bootstrap", true); + IInvokableInstance newInstance = cluster.bootstrap(config); + newInstance.startup(cluster); + // todo: re-add once we fix write survey/join ring = false mode +// withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000), +// () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster))); +// newInstance.nodetoolResult("join").asserts().success(); + newInstance.nodetoolResult("describecms").asserts().success(); // just make sure we're joined, remove later + } + + private static AccordService service() + { + return (AccordService) AccordService.instance(); + } + + private static void awaitEpoch(long epoch) + { + try + { + boolean completed = service().epochReady(Epoch.create(epoch)).await(60, TimeUnit.SECONDS); + Assertions.assertThat(completed) + .describedAs("Epoch %s did not become ready within timeout on %s -> %s", + epoch, FBUtilities.getBroadcastAddressAndPort(), + service().configurationService().getEpochSnapshot(epoch)) + .isTrue(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + private static void awaitLocalSyncNotification(long epoch) + { + try + { + AccordConfigurationService configService = service().configurationService(); + boolean completed = configService.localSyncNotified(epoch).await(5, TimeUnit.SECONDS); + Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s", + epoch, FBUtilities.getBroadcastAddressAndPort()), completed); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + private static long maxEpoch(Cluster cluster) + { + return cluster.stream().mapToLong(node -> node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch())).max().getAsLong(); + } + + private static class StreamListener implements StreamManager.StreamListener + { + private static boolean isRegistered = false; + private static final StreamListener listener = new StreamListener(); + + private final List registered = new ArrayList<>(); + + static synchronized void register() + { + if (isRegistered) + return; + StreamManager.instance.addListener(listener); + isRegistered = true; + } + + public synchronized void onRegister(StreamResultFuture result) + { + registered.add(result); + } + + public synchronized void forSession(Consumer consumer) + { + registered.forEach(future -> { + future.getCoordinator().getAllStreamSessions().forEach(consumer); + }); + } + } + + @Test + public void bootstrapTest() throws Throwable + { + + int originalNodeCount = 2; + int expandedNodeCount = originalNodeCount + 1; + + try (Cluster cluster = Cluster.build().withNodes(originalNodeCount) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0")) + .withConfig(config -> config.with(NETWORK, GOSSIP)) + .start()) + { + long initialMax = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + + node.runOnInstance(() -> { + Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch()); + awaitEpoch(initialMax); + AccordConfigurationService configService = service().configurationService(); + long minEpoch = configService.minEpoch(); + + Assert.assertEquals(initialMax, configService.maxEpoch()); + + for (long epoch = minEpoch; epoch < initialMax; epoch++) + { + awaitEpoch(epoch); + Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch)); + } + + awaitLocalSyncNotification(initialMax); + Assert.assertEquals(EpochSnapshot.completed(initialMax), configService.getEpochSnapshot(initialMax)); + }); + } + + for (IInvokableInstance node : cluster) + { + node.runOnInstance(StreamListener::register); + } + + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))"); + + long schemaChangeMax = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(schemaChangeMax)); + awaitEpoch(schemaChangeMax); + AccordConfigurationService configService = service().configurationService(); + + for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++) + { + awaitLocalSyncNotification(epoch); + Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch)); + } + }); + } + + for (int key = 0; key < 100; key++) + { + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + AccordTestBase.executeWithRetry(cluster, query); + } + + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + Assert.assertTrue(StreamListener.listener.registered.isEmpty()); + }); + } + + bootstrapAndJoinNode(cluster); + long bootstrapMax = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(bootstrapMax)); + Assert.assertEquals(bootstrapMax, ClusterMetadata.current().epoch.getEpoch()); + AccordService service = (AccordService) AccordService.instance(); + awaitEpoch(bootstrapMax); + AccordConfigurationService configService = service.configurationService(); + + awaitLocalSyncNotification(bootstrapMax); + Assert.assertEquals(EpochSnapshot.completed(bootstrapMax), configService.getEpochSnapshot(bootstrapMax)); + }); + } + + InetAddress node3Addr = cluster.get(3).broadcastAddress().getAddress(); + for (IInvokableInstance node : cluster.get(1, 2)) + { + node.runOnInstance(() -> { + + StreamListener.listener.forSession(session -> { + Assert.assertEquals(node3Addr, session.peer.getAddress()); + Assert.assertEquals(0, session.getNumRequests()); + Assert.assertTrue(session.getNumTransfers() > 0); + }); + + awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(safeStore -> { + CommandStore commandStore = safeStore.commandStore(); + Assert.assertEquals(0, commandStore.maxBootstrapEpoch()); + Assert.assertEquals(Timestamp.NONE, getOnlyElement(commandStore.bootstrapBeganAt().keySet())); + Assert.assertEquals(Timestamp.NONE, getOnlyElement(commandStore.safeToRead().keySet())); + })); + }); + } + + cluster.get(3).runOnInstance(() -> { + List> ranges = StorageService.instance.getLocalRanges("ks"); + for (int key = 0; key < 100; key++) + { + UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key); + PartitionKey partitionKey = pk(key, "ks", "tbl"); + if (ranges.stream().anyMatch(range -> range.contains(partitionKey.token()))) + { + UntypedResultSet.Row row = getOnlyElement(result); + Assert.assertEquals(key, row.getInt("c")); + Assert.assertEquals(key, row.getInt("v")); + + awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(safeStore -> { + if (safeStore.ranges().currentRanges().contains(partitionKey)) + { + CommandStore commandStore = safeStore.commandStore(); + Assert.assertTrue(commandStore.maxBootstrapEpoch() > 0); + Assert.assertFalse(commandStore.bootstrapBeganAt().isEmpty()); + Assert.assertFalse(commandStore.safeToRead().isEmpty()); + + Assert.assertEquals(1, commandStore.bootstrapBeganAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + Assert.assertEquals(1, commandStore.safeToRead().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + } + })); + } + else + { + Assert.assertTrue(result.isEmpty()); + } + } + }); + } + } + + @Test + public void moveTest() throws Throwable + { + try (Cluster cluster = Cluster.build().withNodes(3) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0")) + .withConfig(config -> config.with(NETWORK, GOSSIP)) + .start()) + { + long initialMax = maxEpoch(cluster); + long[] tokens = new long[3]; + for (int i=0; i<3; i++) + { + tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens()))); + } + + for (IInvokableInstance node : cluster) + { + + node.runOnInstance(() -> { + Assert.assertEquals(initialMax, ClusterMetadata.current().epoch.getEpoch()); + awaitEpoch(initialMax); + AccordConfigurationService configService = service().configurationService(); + long minEpoch = configService.minEpoch(); + + Assert.assertEquals(initialMax, configService.maxEpoch()); + + for (long epoch = minEpoch; epoch < initialMax; epoch++) + { + awaitEpoch(epoch); + Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch)); + } + + awaitLocalSyncNotification(initialMax); + Assert.assertEquals(EpochSnapshot.completed(initialMax), configService.getEpochSnapshot(initialMax)); + }); + } + + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c))"); + + long schemaChangeMax = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + Assert.assertEquals(schemaChangeMax, ClusterMetadata.current().epoch.getEpoch()); + AccordService service = (AccordService) AccordService.instance(); + awaitEpoch(schemaChangeMax); + AccordConfigurationService configService = service.configurationService(); + + for (long epoch = initialMax + 1; epoch <= schemaChangeMax; epoch++) + { + awaitLocalSyncNotification(epoch); + Assert.assertEquals(EpochSnapshot.completed(epoch), configService.getEpochSnapshot(epoch)); + } + }); + } + + for (int key = 0; key < 100; key++) + { + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + AccordTestBase.executeWithRetry(cluster, query); + } + + long token = ((tokens[1] - tokens[0]) / 2) + tokens[0]; + long preMove = maxEpoch(cluster); + + cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token))); + + long moveMax = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(moveMax)); + Assert.assertEquals(moveMax, ClusterMetadata.current().epoch.getEpoch()); + AccordService service = (AccordService) AccordService.instance(); + awaitEpoch(moveMax); + AccordConfigurationService configService = service.configurationService(); + + awaitLocalSyncNotification(moveMax); + Assert.assertEquals(EpochSnapshot.completed(moveMax), configService.getEpochSnapshot(moveMax)); + }); + } + + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + // validate streaming + List> ranges = StorageService.instance.getLocalRanges("ks"); + TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id; + for (int key = 0; key < 100; key++) + { + DecoratedKey dk = dk(key); + UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key); + if (ranges.stream().anyMatch(range -> range.contains(dk.getToken()))) + { + UntypedResultSet.Row row = getOnlyElement(result); + Assert.assertEquals(key, row.getInt("c")); + Assert.assertEquals(key, row.getInt("v")); + + PartitionKey partitionKey = new PartitionKey("ks", tableId, dk); + + awaitUninterruptiblyAndRethrow(service().node().commandStores().forEach(PreLoadContext.contextFor(partitionKey), + partitionKey.toUnseekable(), moveMax, moveMax, + safeStore -> { + if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) + { + CommandStore commandStore = safeStore.commandStore(); + Assert.assertTrue(commandStore.maxBootstrapEpoch() > 0); + Assert.assertFalse(commandStore.bootstrapBeganAt().isEmpty()); + Assert.assertFalse(commandStore.safeToRead().isEmpty()); + + Assert.assertEquals(1, commandStore.bootstrapBeganAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + Assert.assertEquals(1, commandStore.safeToRead().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + } + })); + } + } + }); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java index 8004a0eef3..a113d69aec 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTest.java @@ -2235,7 +2235,6 @@ public class AccordCQLTest extends AccordTestBase SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 2}"); SHARED_CLUSTER.schemaChange("CREATE TABLE " + currentTable + "1 (k int, c int, v int, primary key (k, c))"); SHARED_CLUSTER.schemaChange("CREATE TABLE " + currentTable + "2 (k int, c int, v int, primary key (k, c))"); - SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe())); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + currentTable + "1 (k, c, v) VALUES (1, 2, 3);", ConsistencyLevel.ALL); SHARED_CLUSTER.coordinator(1).execute("INSERT INTO " + currentTable + "2 (k, c, v) VALUES (2, 2, 4);", ConsistencyLevel.ALL); @@ -2413,7 +2412,6 @@ public class AccordCQLTest extends AccordTestBase SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.org_users ( org_name text, user text, members_version int static, permissions int, PRIMARY KEY (org_name, user) );"); SHARED_CLUSTER.schemaChange("CREATE TABLE demo_ks.user_docs ( user text, doc_id int, title text, org_name text, permissions int, PRIMARY KEY (user, doc_id) );"); - SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe())); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); SHARED_CLUSTER.coordinator(1).execute("INSERT INTO demo_ks.org_users (org_name, user, members_version, permissions) VALUES ('demo', 'blake', 5, 777);\n", ConsistencyLevel.ALL); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java index 6118f765a5..e4bf7ccb7c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordFeatureFlagTest.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.cassandra.schema.Schema; import org.assertj.core.api.Assertions; import org.junit.Test; @@ -38,7 +39,6 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.utils.AssertionUtils; @@ -71,7 +71,7 @@ public class AccordFeatureFlagTest extends TestBaseImpl // The Accord system keyspace should not be present: assertEquals("The Accord system keyspace should not exist", Optional.empty(), cluster.get(1).callOnInstance(() -> Schema.instance.localKeyspaces().get(ACCORD_KEYSPACE_NAME))); - + // Make sure virtual tables don't exist: IIsolatedExecutor.SerializableCallable> hasAccordVirtualTables = () -> SystemViewsKeyspace.instance.tables().stream().filter(t -> t.getClass().equals(AccordVirtualTables.Epoch.class)); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java index 261620705e..9146192ba2 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java @@ -117,7 +117,6 @@ public abstract class AccordTestBase extends TestBaseImpl { for (String ddl : ddls) SHARED_CLUSTER.schemaChange(ddl); - SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe())); // Evict commands from the cache immediately to expose problems loading from disk. SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); @@ -182,7 +181,7 @@ public abstract class AccordTestBase extends TestBaseImpl } // TODO: Retry on preemption may become unnecessary after the Unified Log is integrated. - protected SimpleQueryResult assertRowEqualsWithPreemptedRetry(Cluster cluster, Object[] row, String check, Object... boundValues) + protected static SimpleQueryResult assertRowEqualsWithPreemptedRetry(Cluster cluster, Object[] row, String check, Object... boundValues) { return assertRowWithPreemptedRetry(cluster, QueryResults.builder().row(row).build(), check, boundValues); } @@ -192,7 +191,7 @@ public abstract class AccordTestBase extends TestBaseImpl return assertRowWithPreemptedRetry(cluster, QueryResults.builder().build(), check, boundValues); } - private SimpleQueryResult assertRowWithPreemptedRetry(Cluster cluster, SimpleQueryResult expected, String check, Object... boundValues) + private static SimpleQueryResult assertRowWithPreemptedRetry(Cluster cluster, SimpleQueryResult expected, String check, Object... boundValues) { SimpleQueryResult result = executeWithRetry(cluster, check, boundValues); QueryResultUtil.assertThat(result).isEqualTo(expected); @@ -217,7 +216,7 @@ public abstract class AccordTestBase extends TestBaseImpl } } - protected static SimpleQueryResult executeWithRetry(Cluster cluster, String check, Object... boundValues) + public static SimpleQueryResult executeWithRetry(Cluster cluster, String check, Object... boundValues) { check = wrapInTxn(check); diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java index 196638eb86..0b1cee3e0a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/NewSchemaTest.java @@ -47,7 +47,6 @@ public class NewSchemaTest extends AccordTestBase String table = ks + ".tbl" + i; SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + ks + " WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 1}"); SHARED_CLUSTER.schemaChange(String.format("CREATE TABLE %s (pk blob primary key)", table)); - SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().createEpochFromConfigUnsafe())); SHARED_CLUSTER.forEach(node -> node.runOnInstance(() -> AccordService.instance().setCacheSize(0))); List keys = tokensToKeys(tokens()); diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java index 844a8df368..8ed4556194 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/AbstractPairOfSequencesPaxosSimulation.java @@ -43,7 +43,6 @@ import org.apache.cassandra.distributed.api.LogResult; import org.apache.cassandra.distributed.impl.FileLogAction; import org.apache.cassandra.distributed.impl.Instance; import org.apache.cassandra.io.util.File; -import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.simulator.Action; import org.apache.cassandra.simulator.ActionList; import org.apache.cassandra.simulator.ActionPlan; @@ -191,9 +190,7 @@ abstract class AbstractPairOfSequencesPaxosSimulation extends PaxosSimulation plan = plan.encapsulate(ActionPlan.setUpTearDown( ActionList.of( - cluster.stream().map(i -> simulated.run("Insert Partitions", i, executeForPrimaryKeys(preInsertStmt(), primaryKeys))), - // TODO (now): this is temporary until we have correct epoch handling - cluster.stream().map(i -> simulated.run("Create Accord Epoch", i, () -> AccordService.instance().createEpochFromConfigUnsafe())) + cluster.stream().map(i -> simulated.run("Insert Partitions", i, executeForPrimaryKeys(preInsertStmt(), primaryKeys))) ), ActionList.of( cluster.stream().map(i -> checkErrorLogs(i)), diff --git a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java index 4721a7b4eb..94bc2d937b 100644 --- a/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java +++ b/test/unit/org/apache/cassandra/audit/AuditLoggerTest.java @@ -64,7 +64,6 @@ import org.apache.cassandra.cql3.QueryEvents; import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.utils.JMXServerUtils; import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_AUTHORIZER; @@ -444,7 +443,6 @@ public class AuditLoggerTest extends CQLTester public void testTransactionAuditing() { createTable("CREATE TABLE %s (key int PRIMARY KEY, val int)"); - AccordService.instance().createEpochFromConfigUnsafe(); Session session = sessionNet(); String fqTableName = KEYSPACE + "." + currentTable(); diff --git a/test/unit/org/apache/cassandra/auth/TxnAuthTest.java b/test/unit/org/apache/cassandra/auth/TxnAuthTest.java index 8efd05ca10..d60c3cadc3 100644 --- a/test/unit/org/apache/cassandra/auth/TxnAuthTest.java +++ b/test/unit/org/apache/cassandra/auth/TxnAuthTest.java @@ -36,7 +36,6 @@ import org.apache.cassandra.cql3.statements.TransactionStatement; import org.apache.cassandra.exceptions.UnauthorizedException; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; -import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.transport.messages.ResultMessage; import static org.junit.Assert.assertEquals; @@ -68,7 +67,6 @@ public class TxnAuthTest extends CQLTester public void setUpTest() { createTable("CREATE TABLE %s (k int, v int, PRIMARY KEY(k))"); - AccordService.instance().createEpochFromConfigUnsafe(); } @Test diff --git a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java index 4638970628..360b92ded5 100644 --- a/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java +++ b/test/unit/org/apache/cassandra/cql3/NodeLocalConsistencyTest.java @@ -21,7 +21,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.service.accord.AccordService; import static org.junit.Assert.assertEquals; @@ -97,8 +96,6 @@ public class NodeLocalConsistencyTest extends CQLTester createTable("CREATE TABLE %s (key text, val int, PRIMARY KEY(key))"); QueryProcessor.process(formatQuery("INSERT INTO %s (key, val) VALUES ('foo', 0)"), NODE_LOCAL); - AccordService.instance().createEpochFromConfigUnsafe(); - String query = "BEGIN TRANSACTION\n" + " SELECT * FROM %s WHERE key = 'foo';\n" + "COMMIT TRANSACTION"; diff --git a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java index db84d8fde3..8819a1c7c6 100644 --- a/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java +++ b/test/unit/org/apache/cassandra/cql3/PreparedStatementsTest.java @@ -871,8 +871,6 @@ public class PreparedStatementsTest extends CQLTester private static void updateTxnState() { - //TODO Remove this method once CEP-21 and CEP-15 integrate - AccordService.instance().createEpochFromConfigUnsafe(); AccordService.instance().setCacheSize(0); } } diff --git a/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java b/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java index b7f431dd81..1beaa123a8 100644 --- a/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java +++ b/test/unit/org/apache/cassandra/hints/HintServiceBytemanTest.java @@ -42,7 +42,7 @@ import org.awaitility.Awaitility; import org.jboss.byteman.contrib.bmunit.BMRule; import org.jboss.byteman.contrib.bmunit.BMUnitRunner; -import static org.apache.cassandra.hints.HintsTestUtil.MockFailureDetector; +import org.apache.cassandra.utils.MockFailureDetector; import static org.apache.cassandra.hints.HintsTestUtil.sendHintsAndResponses; import static org.junit.Assert.assertEquals; @@ -52,7 +52,7 @@ public class HintServiceBytemanTest private static final String KEYSPACE = "hints_service_test"; private static final String TABLE = "table"; - private final MockFailureDetector failureDetector = new HintsTestUtil.MockFailureDetector(); + private final MockFailureDetector failureDetector = new MockFailureDetector(); private static TableMetadata metadata; @BeforeClass diff --git a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java index dd0eb5a6ed..1e12d3911e 100644 --- a/test/unit/org/apache/cassandra/hints/HintsServiceTest.java +++ b/test/unit/org/apache/cassandra/hints/HintsServiceTest.java @@ -41,7 +41,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; -import static org.apache.cassandra.hints.HintsTestUtil.MockFailureDetector; +import org.apache.cassandra.utils.MockFailureDetector; import static org.apache.cassandra.hints.HintsTestUtil.sendHintsAndResponses; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java index 727404e6e8..8ab65998c4 100644 --- a/test/unit/org/apache/cassandra/hints/HintsTestUtil.java +++ b/test/unit/org/apache/cassandra/hints/HintsTestUtil.java @@ -24,9 +24,6 @@ import com.google.common.collect.Iterators; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.partitions.AbstractBTreePartition; import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.gms.IFailureDetectionEventListener; -import org.apache.cassandra.gms.IFailureDetector; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MockMessagingService; import org.apache.cassandra.net.MockMessagingSpy; @@ -91,44 +88,4 @@ final class HintsTestUtil } return spy; } - - static class MockFailureDetector implements IFailureDetector - { - boolean isAlive = true; - - public boolean isAlive(InetAddressAndPort ep) - { - return isAlive; - } - - public void interpret(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - - public void report(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - - public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - throw new UnsupportedOperationException(); - } - - public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - throw new UnsupportedOperationException(); - } - - public void remove(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - - public void forceConviction(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java new file mode 100644 index 0000000000..b43209c2cd --- /dev/null +++ b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.accord; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import accord.api.ConfigurationService.EpochReady; +import accord.impl.AbstractConfigurationServiceTest; +import accord.local.Node; +import accord.local.Node.Id; +import accord.topology.Shard; +import accord.topology.Topology; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.ConnectionType; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessageDelivery; +import org.apache.cassandra.net.RequestCallback; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordKeyspace.EpochDiskState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.MockFailureDetector; +import org.apache.cassandra.utils.concurrent.Future; + +import static accord.impl.AbstractConfigurationServiceTest.TestListener; +import static com.google.common.collect.ImmutableSet.of; +import static java.lang.String.format; +import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; +import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; +import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME; +import static org.apache.cassandra.service.accord.AccordKeyspace.EPOCH_METADATA; +import static org.apache.cassandra.service.accord.AccordKeyspace.TOPOLOGIES; +import static org.apache.cassandra.service.accord.AccordKeyspace.loadEpoch; + +public class AccordConfigurationServiceTest +{ + private static final Id ID1 = new Id(1); + private static final Id ID2 = new Id(2); + private static final Id ID3 = new Id(3); + private static final List ID_LIST = ImmutableList.of(ID1, ID2, ID3); + private static final Set ID_SET = ImmutableSet.copyOf(ID_LIST); + private static final TableId TBL1 = TableId.fromUUID(new UUID(0, 1)); + private static final TableId TBL2 = TableId.fromUUID(new UUID(0, 2)); + + private static EndpointMapping mappingForEpoch(long epoch) + { + try + { + EndpointMapping.Builder builder = EndpointMapping.builder(epoch); + builder.add(InetAddressAndPort.getByName("127.0.0.1"), ID1); + builder.add(InetAddressAndPort.getByName("127.0.0.2"), ID2); + builder.add(InetAddressAndPort.getByName("127.0.0.3"), ID3); + return builder.build(); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + private static EndpointMapping mappingForTopology(Topology topology) + { + try + { + EndpointMapping.Builder builder = EndpointMapping.builder(topology.epoch()); + for (Node.Id id : topology.nodes()) + builder.add(InetAddressAndPort.getByName("127.0.0." + id.id), id); + return builder.build(); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + private static class Messaging implements MessageDelivery + { + static class Request + { + final Message message; + final InetAddressAndPort to; + final RequestCallback callback; + + public Request(Message message, InetAddressAndPort to, RequestCallback callback) + { + this.message = message; + this.to = to; + this.callback = callback; + } + } + + final List requests = new ArrayList<>(); + + @Override + public void send(Message message, InetAddressAndPort to) + { + requests.add(new Request(message, to, null)); + } + + @Override + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb) + { + requests.add(new Request(message, to, cb)); + } + + @Override + public void sendWithCallback(Message message, InetAddressAndPort to, RequestCallback cb, ConnectionType specifyConnection) + { + throw new UnsupportedOperationException(); + } + + @Override + public Future> sendWithResult(Message message, InetAddressAndPort to) + { + throw new UnsupportedOperationException(); + } + + @Override + public void respond(V response, Message message) + { + throw new UnsupportedOperationException(); + } + } + + @BeforeClass + public static void beforeClass() throws Throwable + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + ServerTestUtils.daemonInitialization(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), + parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); + } + + @Before + public void setup() + { + Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStore(TOPOLOGIES).truncateBlocking(); + Keyspace.open(ACCORD_KEYSPACE_NAME).getColumnFamilyStore(EPOCH_METADATA).truncateBlocking(); + } + + @Test + public void initialEpochTest() throws Throwable + { + AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector()); + Assert.assertEquals(null, AccordKeyspace.loadEpochDiskState()); + service.start(); + Assert.assertEquals(null, AccordKeyspace.loadEpochDiskState()); + Assert.assertTrue(executeInternal(format("SELECT * FROM %s.%s WHERE epoch=1", ACCORD_KEYSPACE_NAME, TOPOLOGIES)).isEmpty()); + + Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET)); + service.reportTopology(topology1); + loadEpoch(1, (epoch, topology, syncStatus, pendingSync, remoteSync) -> { + Assert.assertEquals(topology1, topology); + Assert.assertTrue(remoteSync.isEmpty()); + }); + Assert.assertEquals(new EpochDiskState(1, 1), service.diskState()); + + service.remoteSyncComplete(ID1, 1); + service.remoteSyncComplete(ID2, 1); + loadEpoch(1, (epoch, topology, syncStatus, pendingSync, remoteSync) -> { + Assert.assertEquals(topology1, topology); + Assert.assertEquals(Sets.newHashSet(ID1, ID2), remoteSync); + }); + } + + @Test + public void loadTest() throws Throwable + { + AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector()); + service.start(); + + Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET)); + service.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1)); + service.reportTopology(topology1); + service.acknowledgeEpoch(EpochReady.done(1)); + service.remoteSyncComplete(ID1, 1); + service.remoteSyncComplete(ID2, 1); + service.remoteSyncComplete(ID3, 1); + + Topology topology2 = new Topology(2, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2))); + service.reportTopology(topology2); + service.acknowledgeEpoch(EpochReady.done(2)); + service.remoteSyncComplete(ID1, 2); + + Topology topology3 = new Topology(3, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2))); + service.reportTopology(topology3); + service.acknowledgeEpoch(EpochReady.done(3)); + + AccordConfigurationService loaded = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector()); + loaded.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1)); + AbstractConfigurationServiceTest.TestListener listener = new AbstractConfigurationServiceTest.TestListener(loaded, true); + loaded.registerListener(listener); + loaded.start(); + + listener.assertNoTruncates(); + listener.assertTopologiesFor(1L, 2L, 3L); + listener.assertTopologyForEpoch(1, topology1); + listener.assertTopologyForEpoch(2, topology2); + listener.assertTopologyForEpoch(3, topology3); + listener.assertSyncsFor(1L, 2L); + listener.assertSyncsForEpoch(1, ID1, ID2, ID3); + listener.assertSyncsForEpoch(2, ID1); + } + + @Test + public void truncateTest() + { + AccordConfigurationService service = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector()); + TestListener serviceListener = new TestListener(service, true); + service.registerListener(serviceListener); + service.start(); + + Topology topology1 = new Topology(1, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, ID_SET)); + service.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1)); + service.reportTopology(topology1); + + Topology topology2 = new Topology(2, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2))); + service.reportTopology(topology2); + + Topology topology3 = new Topology(3, new Shard(AccordTopologyUtils.fullRange("ks"), ID_LIST, of(ID1, ID2))); + service.reportTopology(topology3); + service.truncateTopologiesUntil(3); + Assert.assertEquals(new EpochDiskState(3, 3), service.diskState()); + serviceListener.assertTruncates(3L); + + AccordConfigurationService loaded = new AccordConfigurationService(ID1, new Messaging(), new MockFailureDetector()); + loaded.updateMapping(mappingForEpoch(ClusterMetadata.current().epoch.getEpoch() + 1)); + TestListener loadListener = new TestListener(loaded, true); + loaded.registerListener(loadListener); + loaded.start(); + loadListener.assertTopologiesFor(3L); + } +} diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java index bae5fb144b..358aac687f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java @@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord; import org.junit.BeforeClass; import org.junit.Test; - import org.mockito.Mockito; import accord.api.Agent; @@ -29,6 +28,7 @@ import accord.messages.InformOfTxnId; import accord.messages.SimpleReply; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.Verb; @@ -45,8 +45,11 @@ public class AccordMessageSinkTest } @Test - public void informOfTxn() + public void informOfTxn() throws Throwable { + Node.Id id = new Node.Id(1); + InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0.1"); + EndpointMapping mapping = EndpointMapping.builder(5).add(endpoint, id).build(); // There was an issue where the reply was the wrong verb // see CASSANDRA-18375 InformOfTxnId info = Mockito.mock(InformOfTxnId.class); @@ -54,8 +57,8 @@ public class AccordMessageSinkTest SimpleReply reply = SimpleReply.Ok; MessageDelivery messaging = Mockito.mock(MessageDelivery.class); - AccordMessageSink sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging); - sink.reply(new Node.Id(1), req, reply); + AccordMessageSink sink = new AccordMessageSink(Mockito.mock(Agent.class), messaging, mapping); + sink.reply(id, req, reply); Mockito.verify(messaging).send(Mockito.any(), Mockito.any()); } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 21ee94d2a5..77611f4f1a 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -83,7 +83,6 @@ import org.apache.cassandra.service.accord.api.PartitionKey; import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer; import org.apache.cassandra.service.accord.txn.TxnData; import org.apache.cassandra.service.accord.txn.TxnRead; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; @@ -93,11 +92,6 @@ import static java.lang.String.format; public class AccordTestUtils { - public static Id localNodeId() - { - return EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort()); - } - public static class Commands { public static Command notWitnessed(TxnId txnId, PartialTxn txn) @@ -307,7 +301,7 @@ public class AccordTestUtils { TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table); TokenRange range = TokenRange.fullRange(metadata.keyspace); - Node.Id node = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort()); + Node.Id node = new Id(1); Topology topology = new Topology(1, new Shard(range, Lists.newArrayList(node), Sets.newHashSet(node), Collections.emptySet())); NodeTimeService time = new NodeTimeService() { @@ -361,7 +355,7 @@ public class AccordTestUtils { TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table); TokenRange range = TokenRange.fullRange(metadata.keyspace); - Node.Id node = EndpointMapping.endpointToId(FBUtilities.getBroadcastAddressAndPort()); + Node.Id node = new Id(1); Topology topology = new Topology(1, new Shard(range, Lists.newArrayList(node), Sets.newHashSet(node), Collections.emptySet())); AccordCommandStore store = createAccordCommandStore(node, now, topology, loadExecutor, saveExecutor); store.execute(PreLoadContext.empty(), safeStore -> ((AccordCommandStore)safeStore.commandStore()).setCacheSize(1 << 20)); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java b/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java index 9c8540ebda..0185957139 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java @@ -18,54 +18,177 @@ package org.apache.cassandra.service.accord; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import accord.local.Node.Id; +import accord.topology.Shard; import accord.topology.Topology; -import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.accord.api.PartitionKey; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.membership.NodeState; +import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.DataPlacements; import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse; public class AccordTopologyTest { + private static final Id ID1 = new Id(1); + private static final Id ID2 = new Id(2); + private static final Id ID3 = new Id(3); + private static final List NODE_LIST = ImmutableList.of(ID1, ID2, ID3); + private static final Set NODE_SET = ImmutableSet.copyOf(NODE_LIST); + + private static final InetAddressAndPort EP1 = ep(1); + private static final InetAddressAndPort EP2 = ep(2); + private static final InetAddressAndPort EP3 = ep(3); + + private static final IPartitioner partitioner = Murmur3Partitioner.instance; + private static Tables tables = null; + private static KeyspaceMetadata keyspace = null; + private static final Location LOCATION = new Location("DC1", "RACK1"); + @BeforeClass public static void beforeClass() throws Throwable { DatabaseDescriptor.daemonInitialization(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace("ks", KeyspaceParams.simple(1), - parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks")); - StorageService.instance.initServer(); + tables = Tables.of(parse("CREATE TABLE tbl (k int, c int, v int, primary key (k, c))", "ks").build()); + keyspace = KeyspaceMetadata.create("ks", KeyspaceParams.simple(3), tables); + } + + private static InetAddressAndPort ep(int i) + { + try + { + return InetAddressAndPort.getByAddressOverrideDefaults(InetAddress.getByAddress(new byte[]{127, 0, 0, (byte)i}), 7012); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + private static NodeId nodeId(int id) + { + return new NodeId(id); + } + + private static void addNode(ClusterMetadata.Transformer transformer, int node, Token token) + { + NodeId nodeId = nodeId(node); + InetAddressAndPort ep = ep(node); + NodeAddresses addresses = new NodeAddresses(nodeId.toUUID(), ep, ep, ep); + transformer.register(nodeId, addresses, LOCATION, NodeVersion.CURRENT); + transformer.withNodeState(nodeId, NodeState.JOINED); + transformer.proposeToken(nodeId, Collections.singleton(token)); + } + + private static ClusterMetadata configureCluster(List> ranges, Keyspaces keyspaces) + { + assert ranges.size() == 3; + + IPartitioner partitioner = Murmur3Partitioner.instance; + ClusterMetadata empty = new ClusterMetadata(partitioner); + ClusterMetadata.Transformer transformer = empty.transformer(); + transformer.with(new DistributedSchema(Keyspaces.of(keyspace))); + addNode(transformer, 1, ranges.get(0).right); + addNode(transformer, 2, ranges.get(1).right); + addNode(transformer, 3, ranges.get(2).right); + ClusterMetadata metadata = transformer.build().metadata; + + for (KeyspaceMetadata keyspace : keyspaces) + { + ReplicationParams replication = keyspace.params.replication; + AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication); + DataPlacements.Builder placements = metadata.placements.unbuild(); + DataPlacement placement = strategy.calculateDataPlacement(metadata.epoch, metadata.tokenMap.toRanges(), metadata); + placements.with(replication, placement); + metadata = transformer.with(placements.build()).build().metadata; + } + + return metadata; + } + + private static Token token(long t) + { + return new Murmur3Partitioner.LongToken(t); + } + + private static Range range(Token left, Token right) + { + return new Range<>(left, right); + } + + private static Range range(long left, long right) + { + return range(token(left), token(right)); + } + + /** + * Check converter does the right thing if the ring is constructed with min and max tokens + */ + @Test + public void minMaxTokens() + { + List> ranges = ImmutableList.of(range(partitioner.getMinimumToken(), token(-100)), + range(-100, 100), + range(token(100), partitioner.getMaximumToken())); + Assert.assertEquals(partitioner.getMinimumToken(), ranges.get(0).left); + Assert.assertEquals(partitioner.getMaximumToken(), ranges.get(2).right); + ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); + + Topology topology = AccordTopologyUtils.createAccordTopology(metadata, ks -> true); + Topology expected = new Topology(1, + new Shard(AccordTopologyUtils.minRange("ks", ranges.get(0).right), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.range("ks", ranges.get(1)), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.range("ks", ranges.get(2)), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.maxRange("ks", ranges.get(2).right), NODE_LIST, NODE_SET)); + + Assert.assertEquals(expected, topology); } @Test - public void minMaxTokenTest() + public void wrapAroundRanges() { - IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); - Topology topology = AccordTopologyUtils.createTopology(1); - Assert.assertNotEquals(0, topology.size()); - TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id; - Token minToken = partitioner.getMinimumToken(); - Token maxToken = partitioner.getMaximumToken(); + List> ranges = ImmutableList.of(range(-100, 0), + range(0, 100), + range(100, -100)); -// topology.forKey(new AccordKey.TokenKey(tableId, minToken.minKeyBound())); - topology.forKey(new PartitionKey("ks", tableId, new BufferDecoratedKey(minToken, ByteBufferUtil.bytes(0))).toUnseekable()); -// topology.forKey(new AccordKey.TokenKey(tableId, minToken.maxKeyBound())); -// topology.forKey(new AccordKey.TokenKey(tableId, maxToken.minKeyBound())); - topology.forKey(new PartitionKey("ks", tableId, new BufferDecoratedKey(maxToken, ByteBufferUtil.bytes(0))).toUnseekable()); -// topology.forKey(new AccordKey.TokenKey(tableId, maxToken.maxKeyBound())); + ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace)); + Topology topology = AccordTopologyUtils.createAccordTopology(metadata, ks -> true); + Topology expected = new Topology(1, + new Shard(AccordTopologyUtils.minRange("ks", ranges.get(0).left), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.range("ks", ranges.get(0)), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.range("ks", ranges.get(1)), NODE_LIST, NODE_SET), + new Shard(AccordTopologyUtils.maxRange("ks", ranges.get(2).left), NODE_LIST, NODE_SET)); + + Assert.assertEquals(expected, topology); } } diff --git a/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java b/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java index c35fbce0d4..7054b35066 100644 --- a/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java @@ -18,25 +18,24 @@ package org.apache.cassandra.service.accord; -import org.junit.Assert; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import accord.local.Node; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.utils.CassandraGenerators; +import org.assertj.core.api.Assertions; +import static org.quicktheories.QuickTheory.qt; + +import org.quicktheories.generators.SourceDSL; + public class EndpointMappingTest { - private static final Logger logger = LoggerFactory.getLogger(EndpointMappingTest.class); - @Test public void identityTest() throws Throwable { - InetAddressAndPort endpoint = InetAddressAndPort.getByName("127.0.0.1"); - Node.Id id = EndpointMapping.endpointToId(endpoint); - Assert.assertEquals(endpoint, EndpointMapping.idToEndpoint(id)); - logger.info("{} -> {}", endpoint, id); + qt().forAll(CassandraGenerators.INET_ADDRESS_AND_PORT_GEN, SourceDSL.integers().between(1, Integer.MAX_VALUE).map(Node.Id::new)).checkAssert((endpoint, id) -> { + EndpointMapping mapping = EndpointMapping.builder(1).add(endpoint, id).build(); + Assertions.assertThat(mapping.mappedEndpoint(id)).isEqualTo(endpoint); + }); } } diff --git a/test/unit/org/apache/cassandra/utils/MockFailureDetector.java b/test/unit/org/apache/cassandra/utils/MockFailureDetector.java new file mode 100644 index 0000000000..bee0fbb5b4 --- /dev/null +++ b/test/unit/org/apache/cassandra/utils/MockFailureDetector.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import org.apache.cassandra.gms.IFailureDetectionEventListener; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.locator.InetAddressAndPort; + +public class MockFailureDetector implements IFailureDetector +{ + public boolean isAlive = true; + + public boolean isAlive(InetAddressAndPort ep) + { + return isAlive; + } + + public void interpret(InetAddressAndPort ep) + { + throw new UnsupportedOperationException(); + } + + public void report(InetAddressAndPort ep) + { + throw new UnsupportedOperationException(); + } + + public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) + { + throw new UnsupportedOperationException(); + } + + public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) + { + throw new UnsupportedOperationException(); + } + + public void remove(InetAddressAndPort ep) + { + throw new UnsupportedOperationException(); + } + + public void forceConviction(InetAddressAndPort ep) + { + throw new UnsupportedOperationException(); + } +}