diff --git a/modules/accord b/modules/accord index e896c9c328..a575ab316b 160000 --- a/modules/accord +++ b/modules/accord @@ -1 +1 @@ -Subproject commit e896c9c328d3e8a3b7ddd85381edeaf1b4b8ccb2 +Subproject commit a575ab316b4e79f99009801312579bcbf86451ef diff --git a/src/java/org/apache/cassandra/config/AccordSpec.java b/src/java/org/apache/cassandra/config/AccordSpec.java index 94ff4e946a..8a5801e41f 100644 --- a/src/java/org/apache/cassandra/config/AccordSpec.java +++ b/src/java/org/apache/cassandra/config/AccordSpec.java @@ -143,11 +143,12 @@ public class AccordSpec public String slow_syncpoint_preaccept = "10s"; public String slow_txn_preaccept = "30ms <= p50*2 <= 100ms"; public String slow_read = "30ms <= p50*2 <= 100ms"; - public StringRetryStrategy retry_syncpoint = new StringRetryStrategy("10s*attempts <= 600s"); - public StringRetryStrategy retry_durability = new StringRetryStrategy("10s*attempts <= 600s"); - public StringRetryStrategy retry_bootstrap = new StringRetryStrategy("10s*attempts <= 600s"); - public StringRetryStrategy retry_fetch_min_epoch = new StringRetryStrategy("200ms...1s*attempts <= 1s,retries=3"); - public StringRetryStrategy retry_fetch_topology = new StringRetryStrategy("200ms...1s*attempts <= 1s,retries=100"); + public StringRetryStrategy retry_syncpoint = new StringRetryStrategy("10s*attempt <= 600s"); + public StringRetryStrategy retry_durability = new StringRetryStrategy("10s*attempt <= 600s"); + public StringRetryStrategy retry_bootstrap = new StringRetryStrategy("10s*attempt <= 600s"); + public StringRetryStrategy retry_join_bootstrap = new StringRetryStrategy("30s*attempt,attempts=5"); + public StringRetryStrategy retry_fetch_min_epoch = new StringRetryStrategy("200ms...1s*attempt <= 1s,retries=3"); + public StringRetryStrategy retry_fetch_topology = new StringRetryStrategy("200ms...1s*attempt <= 1s,retries=100"); public StringRetryStrategy retry_journal_index_ready = new StringRetryStrategy("100ms"); public volatile DurationSpec.IntSecondsBound fast_path_update_delay = null; diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java index efe8c390cf..e48a945393 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java @@ -18,10 +18,13 @@ package org.apache.cassandra.db.virtual; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -40,16 +43,21 @@ import javax.annotation.Nullable; import com.google.common.collect.ImmutableSet; +import accord.api.LocalListeners; import accord.coordinate.AbstractCoordination; import accord.coordinate.Coordination; import accord.coordinate.Coordination.CoordinationKind; -import accord.coordinate.Coordinations; import accord.coordinate.PrepareRecovery; import accord.coordinate.tracking.AbstractTracker; +import accord.impl.progresslog.DefaultProgressLog.ModeFlag; +import accord.local.Commands.NotifyWaitingOnPlus; import accord.local.cfk.CommandsForKey.TxnInfo; +import accord.primitives.Range; import accord.primitives.Ranges; import accord.primitives.Routable; import accord.primitives.RoutingKeys; +import accord.topology.Shard; +import accord.topology.Topology; import accord.utils.SortedListMap; import accord.utils.TinyEnumSet; import org.apache.cassandra.db.PartitionPosition; @@ -118,6 +126,7 @@ import org.apache.cassandra.service.accord.AccordCommandStores; import org.apache.cassandra.service.accord.AccordExecutor; import org.apache.cassandra.service.accord.AccordJournal; import org.apache.cassandra.service.accord.AccordKeyspace; +import org.apache.cassandra.service.accord.AccordOperations; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTracing; import org.apache.cassandra.service.accord.AccordTracing.BucketMode; @@ -133,6 +142,7 @@ import org.apache.cassandra.service.accord.api.TokenKey; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.service.consensus.migration.TableMigrationState; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.LocalizeString; import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid; @@ -159,10 +169,12 @@ import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.UNSORTED; import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_ACCORD_DEBUG; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; +// TODO (expected): split into separate classes in own package public class AccordDebugKeyspace extends VirtualKeyspace { public static final String COMMANDS_FOR_KEY = "commands_for_key"; public static final String COMMANDS_FOR_KEY_UNMANAGED = "commands_for_key_unmanaged"; + public static final String COMMAND_STORE_OPS = "command_store_ops"; public static final String CONSTANTS = "constants"; public static final String COORDINATIONS = "coordinations"; public static final String DURABILITY_SERVICE = "durability_service"; @@ -170,8 +182,11 @@ public class AccordDebugKeyspace extends VirtualKeyspace public static final String EXECUTOR_CACHE = "executor_cache"; public static final String EXECUTORS = "executors"; public static final String JOURNAL = "journal"; + public static final String LISTENERS_DEPS = "listeners_deps"; + public static final String LISTENERS_LOCAL = "listeners_local"; public static final String MAX_CONFLICTS = "max_conflicts"; public static final String MIGRATION_STATE = "migration_state"; + public static final String NODE_OPS = "node_ops"; public static final String PROGRESS_LOG = "progress_log"; public static final String REDUNDANT_BEFORE = "redundant_before"; public static final String REJECT_BEFORE = "reject_before"; @@ -182,6 +197,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace public static final String TXN_TRACE = "txn_trace"; public static final String TXN_TRACES = "txn_traces"; public static final String TXN_OPS = "txn_ops"; + public static final String SHARD_EPOCHS = "shard_epochs"; private static final Function TO_STRING = AccordDebugKeyspace::toStringOrNull; @@ -190,16 +206,21 @@ public class AccordDebugKeyspace extends VirtualKeyspace private AccordDebugKeyspace() { super(VIRTUAL_ACCORD_DEBUG, List.of( - new ExecutorsTable(), new CoordinationsTable(), new CommandsForKeyTable(), new CommandsForKeyUnmanagedTable(), + new CommandStoreOpsTable(), + new ConstantsTable(), new DurabilityServiceTable(), new DurableBeforeTable(), new ExecutorCacheTable(), + new ExecutorsTable(), + new ListenersDepsTable(), + new ListenersLocalTable(), new JournalTable(), new MaxConflictsTable(), new MigrationStateTable(), + new NodeOpsTable(), new ProgressLogTable(), new RedundantBeforeTable(), new RejectBeforeTable(), @@ -210,7 +231,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace new TxnPatternTraceTable(), new TxnPatternTracesTable(), new TxnOpsTable(), - new ConstantsTable() + new ShardEpochsTable() )); } @@ -232,8 +253,14 @@ public class AccordDebugKeyspace extends VirtualKeyspace for (CoordinationKind coordinationKind : CoordinationKind.values()) dataSet.row("CoordinationKind", coordinationKind.name()); - for (TxnOpsTable.Op op : TxnOpsTable.Op.values()) - dataSet.row("Op", op.name()).column("description", op.description); + for (TxnOpsTable.TxnOp op : TxnOpsTable.TxnOp.values()) + dataSet.row("TxnOp", op.name()).column("description", op.description); + + for (NodeOpsTable.NodeOp op : NodeOpsTable.NodeOp.values()) + dataSet.row("NodeOp", op.name()).column("description", op.description); + + for (CommandStoreOpsTable.CommandStoreOp op : CommandStoreOpsTable.CommandStoreOp.values()) + dataSet.row("CommandStoreOp", op.name()).column("description", op.description); } @Override @@ -268,7 +295,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - AccordCommandStores commandStores = (AccordCommandStores) AccordService.instance().node().commandStores(); + AccordCommandStores commandStores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores(); // TODO (desired): we can easily also support sorted collection for DESC queries for (AccordExecutor executor : commandStores.executors()) { @@ -307,8 +334,8 @@ public class AccordDebugKeyspace extends VirtualKeyspace "Accord Coordination State", "CREATE TABLE %s (\n" + " txn_id 'TxnIdUtf8Type',\n" + - " kind text,\n" + " coordination_id bigint,\n" + + " kind text,\n" + " description text,\n" + " nodes text,\n" + " nodes_inflight text,\n" + @@ -316,29 +343,44 @@ public class AccordDebugKeyspace extends VirtualKeyspace " participants text,\n" + " replies text,\n" + " tracker text,\n" + - " PRIMARY KEY (txn_id, kind, coordination_id)" + - ')', TxnIdUtf8Type.instance), FAIL, UNSORTED); + " PRIMARY KEY (txn_id, coordination_id, kind)" + + ')', TxnIdUtf8Type.instance), FAIL, SORTED); } @Override public void collect(PartitionsCollector collector) { - Coordinations coordinations = AccordService.instance().node().coordinations(); + List coordinations = new ArrayList<>(); + for (Coordination c : AccordService.unsafeInstance().node().coordinations()) + coordinations.add(c); + + Comparator comparator = CoordinationsTable::compareCoordinations; + if (collector.dataRange().isReversed()) + comparator = comparator.reversed(); + coordinations.sort(comparator); + for (Coordination c : coordinations) { - collector.row(toStringOrNull(c.txnId()), c.kind().toString(), c.coordinationId()) - .lazyCollect(columns -> { - columns.add("nodes", c, Coordination::nodes, TO_STRING) - .add("nodes_inflight", c, Coordination::inflight, TO_STRING) - .add("nodes_contacted", c, Coordination::contacted, TO_STRING) - .add("description", c, Coordination::describe, TO_STRING) - .add("participants", c, Coordination::scope, TO_STRING) - .add("replies", c, Coordination::replies, CoordinationsTable::summarise) - .add("tracker", c, Coordination::tracker, AbstractTracker::summariseTracker); - }); + collector.row(toStringOrNull(c.txnId()), c.coordinationId(), c.kind().toString()) + .lazyCollect(columns -> { + columns.add("nodes", c, Coordination::nodes, TO_STRING) + .add("nodes_inflight", c, Coordination::inflight, TO_STRING) + .add("nodes_contacted", c, Coordination::contacted, TO_STRING) + .add("description", c, Coordination::describe, TO_STRING) + .add("participants", c, Coordination::scope, TO_STRING) + .add("replies", c, Coordination::replies, CoordinationsTable::summarise) + .add("tracker", c, Coordination::tracker, AbstractTracker::summariseTracker); + }); } } + private static int compareCoordinations(Coordination c1, Coordination c2) + { + int cmp = c1.txnId().compareTo(c2.txnId()); + if (cmp == 0) cmp = Long.compare(c1.coordinationId(), c2.coordinationId()); + return cmp; + } + private static String summarise(SortedListMap replies) { return AbstractCoordination.summariseReplies(replies, 60); @@ -382,7 +424,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace TokenKey key = TokenKey.parse((String) partitionKey[0], DatabaseDescriptor.getPartitioner()); List cfks = new CopyOnWriteArrayList<>(); - CommandStores commandStores = AccordService.instance().node().commandStores(); + CommandStores commandStores = AccordService.unsafeInstance().node().commandStores(); AccordService.getBlocking(commandStores.forEach("commands_for_key table query", RoutingKeys.of(key), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { SafeCommandsForKey safeCfk = safeStore.get(key); CommandsForKey cfk = safeCfk.current(); @@ -530,7 +572,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - ShardDurability.ImmutableView view = ((AccordService) AccordService.instance()).shardDurability(); + ShardDurability.ImmutableView view = ((AccordService) AccordService.unsafeInstance()).shardDurability(); while (view.advance()) { @@ -578,7 +620,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - DurableBefore durableBefore = AccordService.instance().node().durableBefore(); + DurableBefore durableBefore = AccordService.unsafeInstance().node().durableBefore(); durableBefore.foldlWithBounds( (entry, ignore, start, end) -> { TableId tableId = (TableId) start.prefix(); @@ -612,7 +654,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - AccordCommandStores stores = (AccordCommandStores) AccordService.instance().node().commandStores(); + AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores(); for (AccordExecutor executor : stores.executors()) { try (AccordExecutor.ExclusiveGlobalCaches cache = executor.lockCaches()) @@ -634,6 +676,99 @@ public class AccordDebugKeyspace extends VirtualKeyspace } } + public static final class ListenersDepsTable extends AbstractLazyVirtualTable + { + private ListenersDepsTable() + { + super(parse(VIRTUAL_ACCORD_DEBUG, LISTENERS_DEPS, + "Accord Txn Dependency Listeners", + "CREATE TABLE %s (\n" + + " command_store_id int,\n" + + " waiting_on 'TxnIdUtf8Type',\n" + + " waiting_until text,\n" + + " waiter 'TxnIdUtf8Type',\n" + + " PRIMARY KEY (command_store_id, waiting_on, waiting_until, waiter)" + + ')', Int32Type.instance), FAIL, UNSORTED, ASC); + } + + @Override + public void collect(PartitionsCollector collector) + { + AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores(); + Object[] pk = collector.singlePartitionKey(); + if (pk != null) + { + CommandStore commandStore = stores.forId((Integer)pk[0]); + if (commandStore != null) + addPartition(commandStore, collector); + return; + } + stores.forAllUnsafe(commandStore -> addPartition(commandStore, collector)); + } + + private void addPartition(CommandStore commandStore, PartitionsCollector collector) + { + collector.partition(commandStore.id()) + .collect(rows -> { + // TODO (desired): support maybe execute immediately with safeStore + AccordService.getBlocking(commandStore.chain((PreLoadContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); })); + }); + } + + private void addRows(SafeCommandStore safeStore, RowsCollector rows) + { + LocalListeners listeners = safeStore.commandStore().unsafeGetListeners(); + for (LocalListeners.TxnListener listener : listeners.txnListeners()) + { + rows.add(listener.waitingOn.toString(), listener.awaitingStatus.name(), listener.waiter.toString()) + .eagerCollect(ignore -> {}); + } + } + } + + public static final class ListenersLocalTable extends AbstractLazyVirtualTable + { + private ListenersLocalTable() + { + super(parse(VIRTUAL_ACCORD_DEBUG, LISTENERS_LOCAL, + "Accord Local Listeners", + "CREATE TABLE %s (\n" + + " command_store_id int,\n" + + " waiting_on 'TxnIdUtf8Type',\n" + + " waiter text,\n" + + " PRIMARY KEY (command_store_id, waiting_on, waiter)" + + ')', Int32Type.instance), FAIL, UNSORTED, ASC); + } + + @Override + public void collect(PartitionsCollector collector) + { + AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores(); + Object[] pk = collector.singlePartitionKey(); + if (pk != null) + { + CommandStore commandStore = stores.forId((Integer)pk[0]); + if (commandStore != null) + addPartition(commandStore, collector); + return; + } + stores.forAllUnsafe(commandStore -> addPartition(commandStore, collector)); + } + + private void addPartition(CommandStore commandStore, PartitionsCollector collector) + { + collector.partition(commandStore.id()) + .collect(rows -> { + // TODO (desired): support maybe execute immediately with safeStore + LocalListeners listeners = commandStore.unsafeGetListeners(); + for (LocalListeners.Registered listener : listeners.complexListeners()) + rows.add(listener.waitingOn().toString(), listener.waiting().toString()) + .eagerCollect(ignore -> {}); + + }); + } + } + public static final class MaxConflictsTable extends AbstractLazyVirtualTable { private MaxConflictsTable() @@ -653,7 +788,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - CommandStores commandStores = AccordService.instance().node().commandStores(); + CommandStores commandStores = AccordService.unsafeInstance().node().commandStores(); for (CommandStore commandStore : commandStores.all()) { @@ -800,7 +935,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - CommandStores commandStores = AccordService.instance().node().commandStores(); + CommandStores commandStores = AccordService.unsafeInstance().node().commandStores(); for (CommandStore commandStore : commandStores.all()) { DefaultProgressLog.ImmutableView view = ((DefaultProgressLog) commandStore.unsafeProgressLog()).immutableView(); @@ -874,7 +1009,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override public void collect(PartitionsCollector collector) { - CommandStores commandStores = AccordService.instance().node().commandStores(); + CommandStores commandStores = AccordService.unsafeInstance().node().commandStores(); for (CommandStore commandStore : commandStores.all()) { @@ -929,7 +1064,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace @Override protected void collect(PartitionsCollector collector) { - CommandStores commandStores = AccordService.instance().node().commandStores(); + CommandStores commandStores = AccordService.unsafeInstance().node().commandStores(); for (CommandStore commandStore : commandStores.all()) { RejectBefore rejectBefore = commandStore.unsafeGetRejectBefore(); @@ -1399,7 +1534,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace { AccordService accord; { - IAccordService iaccord = AccordService.instance(); + IAccordService iaccord = AccordService.unsafeInstance(); if (!iaccord.isEnabled()) return; @@ -1599,7 +1734,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace public static final class TxnOpsTable extends AbstractMutableLazyVirtualTable { // TODO (expected): test each of these operations - enum Op + enum TxnOp { LOCALLY_ERASE_VESTIGIAL("USE WITH CAUTION: Move the command to the vestigial status, erasing its contents. This has distributed state machine implications."), LOCALLY_INVALIDATE("USE WITH CAUTION: Move the command to the invalidated status, erasing its contents. This has distributed state machine implications."), @@ -1612,7 +1747,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace final String description; - Op(String description) + TxnOp(String description) { this.description = description; } @@ -1641,7 +1776,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace TxnId txnId = TxnId.parse((String) partitionKeys[0]); int commandStoreId = (Integer) clusteringKeys[0]; - Op op = tryParse(values[0], true, Op.class, Op::valueOf); + TxnOp op = tryParse(values[0], true, TxnOp.class, TxnOp::valueOf); switch (op) { @@ -1655,7 +1790,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace case TRY_EXECUTE: run(txnId, commandStoreId, safeStore -> { SafeCommand safeCommand = safeStore.unsafeGet(txnId); - Commands.maybeExecute(safeStore, safeCommand, true, true); + Commands.maybeExecute(safeStore, safeCommand, safeCommand.current(), true, true, NotifyWaitingOnPlus.adapter(null, true, true)); return AsyncChains.success(null); }); break; @@ -1703,7 +1838,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace Command command = safeCommand.current(); if (command == null) throw new InvalidRequestException(txnId + " not known"); - Node node = AccordService.instance().node(); + Node node = AccordService.unsafeInstance().node(); AsyncResult.Settable result = new AsyncResults.SettableResult<>(); BiConsumer, AsyncResult.Settable> consumer = apply.apply(command); if (command.route() == null) @@ -1723,7 +1858,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace private void fetch(TxnId txnId, Timestamp executeAtIfKnown, Route route, AsyncResult.Settable result) { - Node node = AccordService.instance().node(); + Node node = AccordService.unsafeInstance().node(); FetchData.fetchSpecific(Known.Apply, node, txnId, executeAtIfKnown, route, route.withHomeKey(), LatentStoreSelector.standard(), (success, fail) -> { if (fail != null) result.setFailure(fail); else result.setSuccess(null); @@ -1732,7 +1867,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace private void recover(TxnId txnId, @Nullable Route route, AsyncResult.Settable result) { - Node node = AccordService.instance().node(); + Node node = AccordService.unsafeInstance().node(); if (Route.isFullRoute(route)) { PrepareRecovery.recover(node, node.someSequentialExecutor(), txnId, NotKnownToBeInvalid, (FullRoute) route, null, LatentStoreSelector.standard(), (success, fail) -> { @@ -1742,7 +1877,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace } else { - MaybeRecover.maybeRecover(node, txnId, NotKnownToBeInvalid, route, ProgressToken.NONE, LatentStoreSelector.standard(), (success, fail) -> { + MaybeRecover.maybeRecover(node, txnId, NotKnownToBeInvalid, route, ProgressToken.NONE, true, LatentStoreSelector.standard(), (success, fail) -> { if (fail != null) result.setFailure(fail); else result.setSuccess(null); }); @@ -1751,7 +1886,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace private void run(TxnId txnId, int commandStoreId, Function> apply) { - AccordService accord = (AccordService) AccordService.instance(); + AccordService accord = (AccordService) AccordService.unsafeInstance(); AccordService.getBlocking(accord.node() .commandStores() .forId(commandStoreId) @@ -1770,11 +1905,176 @@ public class AccordDebugKeyspace extends VirtualKeyspace } } + /** + * Write-only virtual table for updating Accord node states + */ + public static final class NodeOpsTable extends AbstractMutableLazyVirtualTable + { + // TODO (expected): test each of these operations + enum NodeOp + { + MARK_STALE("Mark the node as offline for an indeterminate period. Peers will be allowed to garbage collect without involving the marked node, and when it comes online it will need to rejoin the transaction log."), + UNMARK_STALE("Peers will no longer be allowed to garbage collect without involving the node."), + MARK_HARD_REMOVED("EMERGENCY USE ONLY: Mark the node as PERMANENTLY offline. It must be guaranteed not to respond at any future date. If a quorum is marked HARD_REMOVED, the remaining replicas in a shard may collectively make progress, though consistency violations are possible."), + FORCE_MARK_HARD_REMOVED("EMERGENCY USE ONLY: Mark the node as PERMANENTLY offline WHETHER OR NOT it has left the most recent topology. It must be guaranteed not to respond at any future date. If a quorum is marked HARD_REMOVED, the remaining replicas in a shard may collectively make progress, though consistency violations are possible."), + ; + + final String description; + + NodeOp(String description) + { + this.description = description; + } + } + private NodeOpsTable() + { + super(parse(VIRTUAL_ACCORD_DEBUG, NODE_OPS, + "Update Accord Node State", + "CREATE TABLE %s (\n" + + " node_id int,\n" + + " op text," + + " PRIMARY KEY (node_id)" + + ')', Int32Type.instance), FAIL, UNSORTED); + } + + @Override + protected void collect(PartitionsCollector collector) + { + throw new UnsupportedOperationException(NODE_OPS + " is a write-only table"); + } + + @Override + protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, ColumnMetadata[] columns, Object[] values) + { + NodeId nodeId = new NodeId((Integer) partitionKeys[0]); + Set nodeIds = Collections.singleton(nodeId); + NodeOp op = tryParse(values[0], true, NodeOp.class, NodeOp::valueOf); + + switch (op) + { + default: throw new UnhandledEnum(op); + case MARK_STALE: + AccordOperations.instance.accordMarkStale(nodeIds); + break; + case UNMARK_STALE: + AccordOperations.instance.accordMarkRejoining(nodeIds); + break; + case MARK_HARD_REMOVED: + AccordOperations.instance.accordMarkHardRemoved(nodeIds, false); + break; + case FORCE_MARK_HARD_REMOVED: + AccordOperations.instance.accordMarkHardRemoved(nodeIds, true); + break; + } + } + } + + /** + * Write-only virtual table for updating Accord node states + */ + public static final class CommandStoreOpsTable extends AbstractMutableLazyVirtualTable + { + // TODO (expected): test each of these operations + enum CommandStoreOp + { + SET_PROGRESS_LOG_MODE("Set the specified progress log mode."), + UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."), + TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."), + ; + + final String description; + + CommandStoreOp(String description) + { + this.description = description; + } + } + private CommandStoreOpsTable() + { + super(parse(VIRTUAL_ACCORD_DEBUG, COMMAND_STORE_OPS, + "Update Accord CommandStore State", + "CREATE TABLE %s (\n" + + " command_store_id int,\n" + + " op text," + + " param text," + + " PRIMARY KEY (command_store_id)" + + ')', Int32Type.instance), FAIL, UNSORTED); + } + + @Override + protected void collect(PartitionsCollector collector) + { + throw new UnsupportedOperationException(COMMAND_STORE_OPS + " is a write-only table"); + } + + @Override + protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, ColumnMetadata[] columns, Object[] values) + { + int commandStoreId = (Integer) partitionKeys[0]; + + CommandStoreOp op = null; + String param = null; + for (int i = 0 ; i < columns.length ; ++i) + { + switch (columns[i].name.toString()) + { + default: throw new IllegalArgumentException("Unknown column " + columns[i].name.toString()); + case "op": + op = tryParse(values[i], true, CommandStoreOp.class, CommandStoreOp::valueOf); + break; + case "param": + param = (String) values[i]; + break; + } + } + + if (op == null) + throw new IllegalArgumentException("Must specify 'op'"); + + final Function> function; + switch (op) + { + default: throw new UnhandledEnum(op); + case SET_PROGRESS_LOG_MODE: + case UNSET_PROGRESS_LOG_MODE: + if (param == null) + throw new IllegalArgumentException("Must specify 'param' for " + op); + ModeFlag mode = tryParse(param, true, ModeFlag.class, ModeFlag::valueOf); + boolean set = op == CommandStoreOp.SET_PROGRESS_LOG_MODE; + function = commandStore -> { + DefaultProgressLog progressLog = ((DefaultProgressLog)commandStore.unsafeProgressLog()); + if (set) progressLog.setMode(mode); + else progressLog.unsetMode(mode); + return AsyncResults.success(null); + }; + break; + case TRY_EXECUTE_LISTENING: + if (param != null) + throw new IllegalArgumentException("'param' is not supported for " + op); + function = CommandStore::operatorTryToExecuteListeningTxns; + break; + } + + AsyncResult result; + if (commandStoreId < 0) + { + List> results = new ArrayList<>(); + AccordService.unsafeInstance().node() + .commandStores() + .forAllUnsafe(commandStore -> results.add(function.apply(commandStore))); + result = AsyncResults.allOf(results); + } + else + { + result = function.apply(AccordService.unsafeInstance().node().commandStores().forId(commandStoreId)); + } + + AccordService.getBlocking(result); + } + } + public static class TxnBlockedByTable extends AbstractLazyVirtualTable { - enum Reason - {Self, Txn, Key} - protected TxnBlockedByTable() { super(parse(VIRTUAL_ACCORD_DEBUG, TXN_BLOCKED_BY, @@ -1806,7 +2106,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace partition.collect(rows -> { try { - DebugBlockedTxns.visit(AccordService.instance(), txnId, maxDepth, collector.deadlineNanos(), txn -> { + DebugBlockedTxns.visit(AccordService.unsafeInstance(), txnId, maxDepth, collector.deadlineNanos(), txn -> { String keyStr = txn.blockedViaKey == null ? "" : txn.blockedViaKey.toString(); String txnIdStr = txn.txnId == null || txn.txnId.equals(txnId) ? "" : txn.txnId.toString(); rows.add(txn.commandStoreId, txn.depth, keyStr, txnIdStr) @@ -1824,6 +2124,128 @@ public class AccordDebugKeyspace extends VirtualKeyspace } } + // TODO (desired): epoch/token filtering + // TODO (expected): report command_store_ids for each shard + public static final class ShardEpochsTable extends AbstractLazyVirtualTable + { + static class ShardAndEpochs + { + final long startEpoch; + final Shard shard; + long endEpoch; + + ShardAndEpochs(long startEpoch, Shard shard) + { + this.startEpoch = this.endEpoch = startEpoch; + this.shard = shard; + } + } + + private ShardEpochsTable() + { + super(parse(VIRTUAL_ACCORD_DEBUG, SHARD_EPOCHS, + "Accord per-CommandStore Shard Epochs", + "CREATE TABLE %s (\n" + + " table_id text,\n" + + " token_start 'TokenUtf8Type',\n" + + " epoch_start bigint,\n" + + " epoch_end bigint,\n" + + " peers text,\n" + + " peers_fast_path text,\n" + + " peers_hard_removed text,\n" + + " quorum_simple int,\n" + + " quorum_fast int,\n" + + " quorum_fast_privileged_deps int,\n" + + " quorum_fast_privileged_nodeps int,\n" + + " token_end 'TokenUtf8Type',\n" + + " PRIMARY KEY (table_id, token_start, epoch_start)" + + ')', UTF8Type.instance), FAIL, ASC); + } + + @Override + public void collect(PartitionsCollector collector) + { + IAccordService service = AccordService.unsafeInstance(); + List snapshot = service.node().topology().topologySnapshot(); + Map>> tableIdLookup = new HashMap<>(); + + { + TableId filterTableId = null; + Object[] pk = collector.singlePartitionKey(); + if (pk != null) + filterTableId = TableId.fromString((String)pk[0]); + + TableId prevTableId = null; + Map> startLookup = null; + for (Topology topology : snapshot) + { + for (Shard shard : topology.shards()) + { + Range range = shard.range; + TableId tableId = (TableId) range.prefix(); + if (filterTableId != null && !filterTableId.equals(tableId)) + continue; + + if (!tableId.equals(prevTableId)) + { + startLookup = tableIdLookup.computeIfAbsent(tableId, ignore -> new HashMap<>()); + prevTableId = tableId; + } + List shards = startLookup.computeIfAbsent((TokenKey) range.start(), ignore -> new ArrayList<>()); + + if (!shards.isEmpty()) + { + ShardAndEpochs prev = shards.get(shards.size() - 1); + if (prev.endEpoch + 1 == topology.epoch() && prev.shard.equals(shard)) + { + ++prev.endEpoch; + continue; + } + } + + shards.add(new ShardAndEpochs(topology.epoch(), shard)); + } + } + } + + List tableIds = new ArrayList<>(tableIdLookup.keySet()); + tableIds.sort(Comparator.comparing(TableId::toString)); + + List startKeys = new ArrayList<>(); + for (TableId tableId : tableIds) + { + collector.partition(tableId.toString()).collect(rows -> { + + startKeys.clear(); + Map> startLookup = tableIdLookup.get(tableId); + startKeys.addAll(startLookup.keySet()); + startKeys.sort(TokenKey::compareTo); + for (TokenKey start : startKeys) + { + for (ShardAndEpochs shardsAndEpochs : startLookup.get(start)) + { + Shard shard = shardsAndEpochs.shard; + + rows.add(start.token().toString(), shardsAndEpochs.startEpoch) + .lazyCollect(columns -> { + columns.add("epoch_end", shardsAndEpochs.endEpoch) + .add("peers", shard.nodes, TO_STRING) + .add("peers_fast_path", shard.notInFastPath, shard.nodes::without, TO_STRING) + .add("peers_hard_removed", shard.hardRemoved, TO_STRING) + .add("quorum_simple", (int)shard.slowQuorumSize) + .add("quorum_fast", (int)shard.simpleFastQuorumSize) + .add("quorum_fast_privileged_deps", (int)shard.privilegedWithDepsFastQuorumSize) + .add("quorum_fast_privileged_nodeps", (int)shard.privilegedWithoutDepsFastQuorumSize) + .add("token_end", shard.range.end().suffix(), TO_STRING); + + }); + } + } + }); + } + } + } + private static String printToken(RoutingKey routingKey) { TokenKey key = (TokenKey) routingKey; @@ -1858,12 +2280,21 @@ public class AccordDebugKeyspace extends VirtualKeyspace private static String toStringOrNull(Object o) { - if (o == null) + return toStringOrNull(o, Object::toString); + } + + private static String toStringOrNull(V v, Function ifNotNull) + { + if (v == null) return null; try { - return Objects.toString(o); + Object o = ifNotNull.apply(v); + if (o == null) + return null; + + return o.toString(); } catch (Throwable t) { @@ -1913,7 +2344,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace private static AccordTracing tracing() { - return ((AccordAgent)AccordService.instance().agent()).tracing(); + return ((AccordAgent)AccordService.unsafeInstance().agent()).tracing(); } private static > E tryParse(Object input, boolean toUpperCase, Class clazz, Function valueOf) diff --git a/src/java/org/apache/cassandra/db/virtual/AccordDebugRemoteKeyspace.java b/src/java/org/apache/cassandra/db/virtual/AccordDebugRemoteKeyspace.java index 8698b3ccd8..4dbaef95a3 100644 --- a/src/java/org/apache/cassandra/db/virtual/AccordDebugRemoteKeyspace.java +++ b/src/java/org/apache/cassandra/db/virtual/AccordDebugRemoteKeyspace.java @@ -26,6 +26,6 @@ public class AccordDebugRemoteKeyspace extends RemoteToLocalVirtualKeyspace public AccordDebugRemoteKeyspace(String name, VirtualKeyspace wrap) { - super(name, wrap); + super(name, wrap, vt -> !vt.name().equals(AccordDebugKeyspace.NODE_OPS)); } } diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java index d48cf05349..1ef45029e3 100644 --- a/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java +++ b/src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java @@ -43,7 +43,7 @@ public final class VirtualKeyspaceRegistry VirtualKeyspace previous = virtualKeyspaces.put(keyspace.name(), keyspace); // some tests choose to replace the keyspace, if so make sure to cleanup tables as well if (previous != null) - previous.tables().forEach(t -> virtualTables.remove(t)); + previous.tables().forEach(t -> virtualTables.remove(t.metadata().id)); keyspace.tables().forEach(t -> virtualTables.put(t.metadata().id, t)); } diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java index e2520659b8..1cf1a81fdd 100644 --- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java +++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java @@ -63,7 +63,7 @@ import org.apache.cassandra.utils.FastByteOperations; * need to sometimes return a port and sometimes not. * */ -public final class InetAddressAndPort extends InetSocketAddress implements Comparable, Serializable +public final class InetAddressAndPort extends InetSocketAddress implements Comparable, Serializable, Endpoint { private static final long serialVersionUID = 0; private static final Logger logger = LoggerFactory.getLogger(InetAddressAndPort.class); @@ -382,6 +382,12 @@ public final class InetAddressAndPort extends InetSocketAddress implements Compa return defaultPort; } + @Override + public InetAddressAndPort endpoint() + { + return this; + } + public static final class MetadataSerializer implements org.apache.cassandra.tcm.serialization.MetadataSerializer { public static final MetadataSerializer serializer = new MetadataSerializer(); diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index ca6770998e..4bebca17e4 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -192,7 +192,7 @@ public class RepairJob extends AsyncFuture implements Runnable requireAllEndpoints = true; } logger.info("{} {}.{} starting accord repair, require all endpoints {}", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily, requireAllEndpoints); - AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints); + AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints); return repair.repair(taskExecutor).flatMap(accordRepairResult -> { // Propagate the HLC discovered during Accord repair to Paxos so Paxos doesn't use ballots < Accord has already used if (accordRepairResult.maxHlc != IAccordService.NO_HLC) diff --git a/src/java/org/apache/cassandra/service/RetryStrategy.java b/src/java/org/apache/cassandra/service/RetryStrategy.java index 120768ad68..385842e8cd 100644 --- a/src/java/org/apache/cassandra/service/RetryStrategy.java +++ b/src/java/org/apache/cassandra/service/RetryStrategy.java @@ -276,6 +276,13 @@ public class RetryStrategy implements WaitStrategy default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\''); case "retries": retries = Integer.parseInt(value); + if (retries < 0) + throw new IllegalArgumentException("retries must be non-negative (retries=" + value + " supplied)"); + break; + case "attempts": + retries = Integer.parseInt(value); + if (retries < 0) + throw new IllegalArgumentException("Must permit at least one attempt (attempts=" + value + " supplied)"); break; case "rnd": if (randomizer != null) diff --git a/src/java/org/apache/cassandra/service/TimeoutStrategy.java b/src/java/org/apache/cassandra/service/TimeoutStrategy.java index aebbf3d506..7554dcf215 100644 --- a/src/java/org/apache/cassandra/service/TimeoutStrategy.java +++ b/src/java/org/apache/cassandra/service/TimeoutStrategy.java @@ -85,7 +85,7 @@ public class TimeoutStrategy implements WaitStrategy static final Pattern WAIT = Pattern.compile( "\\s*(?0|[0-9]+[mu]?s)" + "|\\s*((p(?[0-9]+)(\\((?r|w|rw|wr)\\))?)?|(?0|[0-9]+[mu]?s))" + - "\\s*(([*]\\s*(?[0-9.]+))?\\s*(?[*^]\\s*attempts)?)?\\s*"); + "\\s*(([*]\\s*(?[0-9.]+))?\\s*(?[*^]\\s*attempts?)?)?\\s*"); static final Pattern TIME = Pattern.compile( "0|[0-9]+[mu]?s"); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java index 55138051ca..d47f0ed0c3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStore.java @@ -474,7 +474,7 @@ public class AccordCommandStore extends CommandStore @Override protected void ensureDurable(Ranges ranges, RedundantBefore onCommandStoreDurable) { - if (!CommandsForKey.reportLinearizabilityViolations()) + if (node().isReplaying()) return; long reportId = nextDurabilityLoggingId.incrementAndGet(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java index 95f0377f25..6d06fd47f2 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java +++ b/src/java/org/apache/cassandra/service/accord/AccordCommandStores.java @@ -191,7 +191,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize return Arrays.asList(executors.clone()); } - public void waitForQuiescense() + public void waitForQuiescence() { for (AccordExecutor executor : this.executors) executor.waitForQuiescence(); diff --git a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java index 80e3296f56..bca1cad65f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java +++ b/src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java @@ -34,13 +34,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import accord.api.Agent; +import accord.api.TopologySorter; +import accord.coordinate.tracking.NotifyTracker; +import accord.coordinate.tracking.RequestStatus; import accord.impl.AbstractConfigurationService; import accord.local.Node; import accord.primitives.Ranges; -import accord.topology.Shard; +import accord.topology.Topologies; import accord.topology.Topology; import accord.utils.Invariants; -import accord.utils.SortedListSet; import accord.utils.async.AsyncResult; import accord.utils.async.AsyncResults; import org.agrona.collections.LongArrayList; @@ -48,8 +50,6 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.concurrent.Shutdownable; import org.apache.cassandra.concurrent.Stage; -import org.apache.cassandra.gms.FailureDetector; -import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessageDelivery; import org.apache.cassandra.net.MessagingService; @@ -68,24 +68,25 @@ import org.slf4j.LoggerFactory; import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord; import static org.apache.cassandra.utils.Simulate.With.MONITORS; -// TODO (desired): listen to FailureDetector and rearrange fast path accordingly @Simulate(with=MONITORS) -public class AccordConfigurationService extends AbstractConfigurationService implements AccordEndpointMapper, AccordSyncPropagator.Listener, Shutdownable +public class AccordConfigurationService extends AbstractConfigurationService implements AccordSyncPropagator.Listener, Shutdownable { public static final Logger logger = LoggerFactory.getLogger(AccordConfigurationService.class); + private final AccordSyncPropagator syncPropagator; - public final WatermarkCollector watermarkCollector; + private final WatermarkCollector watermarkCollector; + private final AccordEndpointMapper endpointMapper; private enum State { INITIALIZED, STARTED, SHUTDOWN } @GuardedBy("this") private State state = State.INITIALIZED; - private volatile EndpointMapping mapping = EndpointMapping.EMPTY; public enum SyncStatus { NOT_STARTED, NOTIFYING, COMPLETED } static class EpochState extends AbstractConfigurationService.AbstractEpochState { + private NotifyTracker notifyTracker; private volatile SyncStatus syncStatus = SyncStatus.NOT_STARTED; protected final AsyncResult.Settable localSyncNotified = AsyncResults.settable(); @@ -94,11 +95,29 @@ public class AccordConfigurationService extends AbstractConfigurationService received() @@ -140,17 +159,18 @@ public class AccordConfigurationService extends AbstractConfigurationService removedNodes = mapping.removedNodes(); - for (Map.Entry e : removedNodes.entrySet()) - onNodeRemoved(e.getValue(), currentTopology(), e.getKey()); + endpointMapper.removedNodes().forEach((removed, removedIn) -> { + onNodeRemoved(removedIn, new Node.Id(removed.id)); + }); } @Override @@ -200,28 +218,9 @@ public class AccordConfigurationService extends AbstractConfigurationService this.mapping.epoch()) - this.mapping = mapping; - } - public synchronized void updateMapping(ClusterMetadata metadata) { - updateMapping(AccordTopology.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory)); + endpointMapper.updateMapping(metadata); } private void reportMetadata(ClusterMetadata metadata) @@ -234,14 +233,6 @@ public class AccordConfigurationService extends AbstractConfigurationService stillLiveNodes = metadata.directory.states.entrySet() .stream() @@ -264,24 +255,14 @@ public class AccordConfigurationService extends AbstractConfigurationService= minEpoch) - onNodeRemoved(topology.epoch(), previous, removedNode); + onNodeRemoved(topology.epoch(), removedNode); } } - private static boolean shareShard(Topology current, Node.Id target, Node.Id self) - { - for (Shard shard : current.shards()) - { - if (!shard.contains(target)) continue; - if (shard.contains(self)) return true; - } - return false; - } - - public void onNodeRemoved(long epoch, Topology current, Node.Id removed) + public void onNodeRemoved(long epoch, Node.Id removed) { syncPropagator.onNodesRemoved(removed); - // TODO (now): it seems to be incorrect to mark remote syncs complete if/when node got removed. + // TODO (desired): this is test only; make integration cleaner for (long oldEpoch : nonCompletedEpochsBefore(epoch)) receiveRemoteSyncCompletePreListenerNotify(removed, oldEpoch); @@ -358,7 +339,7 @@ public class AccordConfigurationService extends AbstractConfigurationService= epoch) - onResult.accept(Success, null); + onResult.accept(SUCCESS, null); else onResult.accept(null, t); } else { topologyRange.forEach(this::reportTopology, epoch, 1); - onResult.accept(Success, null); + onResult.accept(SUCCESS, null); } }); }); @@ -440,39 +421,33 @@ public class AccordConfigurationService extends AbstractConfigurationService notify = SortedListSet.allOf(topology.nodes()); - notify.remove(localId); - syncPropagator.reportSyncComplete(epoch, notify, localId); + syncPropagator.reportCoordinationReady(epoch, topology.nodes(), localId); } @Override public synchronized void onEndpointAck(Node.Id id, long epoch) - { - } - - @Override - public void onComplete(long epoch) { if (epochs.wasTruncated(epoch)) return; @@ -480,13 +455,21 @@ public class AccordConfigurationService extends AbstractConfigurationService removedNodes(); + + NodeStatus nodeStatus(Node.Id id); + default void updateMapping(ClusterMetadata metadata) {} } diff --git a/src/java/org/apache/cassandra/service/accord/AccordFastPath.java b/src/java/org/apache/cassandra/service/accord/AccordFastPath.java index 2afc16f9fe..e9ea757282 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFastPath.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFastPath.java @@ -25,7 +25,7 @@ import java.util.Objects; import accord.local.Node; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.util.DataInputPlus; @@ -235,14 +235,14 @@ public class AccordFastPath implements MetadataValue return lastModified; } - public ImmutableSet unavailableIds() + public SortedArrayList unavailableIds() { - ImmutableSet.Builder builder = ImmutableSet.builder(); - info.entrySet().stream() - .filter(entry -> entry.getValue().status.isUnavailable()) - .map(Map.Entry::getKey) - .forEach(builder::add); - return builder.build(); + // TODO (expected): why don't we save this? + Node.Id[] ids = info.entrySet().stream() + .filter(entry -> entry.getValue().status.isUnavailable()) + .map(Map.Entry::getKey) + .toArray(Node.Id[]::new); + return SortedArrayList.ofUnsorted(ids); } public static final MetadataSerializer serializer = new MetadataSerializer() diff --git a/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java index 4e214e664f..12a706a442 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java @@ -222,14 +222,14 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi @Override public void onAlive(InetAddressAndPort endpoint, EndpointState state) { - Node.Id node = configService.mappedIdOrNull(endpoint); + Node.Id node = configService.endpointMapper().mappedIdOrNull(endpoint); if (node != null) onAlive(node); } @Override public void onDead(InetAddressAndPort endpoint, EndpointState state) { - Node.Id node = configService.mappedIdOrNull(endpoint); + Node.Id node = configService.endpointMapper().mappedIdOrNull(endpoint); if (node != null) onDead(node); } } @@ -335,7 +335,7 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi } @Override - public AsyncResult onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync) + public AsyncResult onTopologyUpdate(Topology topology) { updatePeers(topology); return SUCCESS; diff --git a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java index 0d26c56b56..84880f2e73 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java @@ -25,11 +25,15 @@ import java.util.Map; import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import accord.api.Data; import accord.api.DataStore; import accord.api.Query; import accord.api.Read; import accord.api.Update; +import accord.coordinate.tracking.AbstractTracker; import accord.impl.AbstractFetchCoordinator; import accord.local.CommandStore; import accord.local.Node; @@ -80,6 +84,7 @@ import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamManager.StreamListener { + private static final Logger logger = LoggerFactory.getLogger(AccordFetchCoordinator.class); private static final Query noopQuery = (txnId, executeAt, keys, data, read, update) -> null; public static class StreamData implements Data @@ -101,7 +106,6 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements { TimeUUID.Serializer.instance.serialize(info.planId, out); out.writeBoolean(info.hasData); - } public SessionInfo deserialize(DataInputPlus in) throws IOException @@ -215,19 +219,28 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements Invariants.nonNull(future); Invariants.require(this.future == null, "future was not null: %s", this.future); this.future = future; + logger.info("StreamFuture for plan {} received for bootstrap of {} from {}", planId, range, from); maybeListen(); } private void maybeListen() { + // TODO (required): if for some reason the stream isn't initiated this hangs if (range == null || future == null) return; Invariants.nonNull(from); - future.addCallback((state, fail) -> { - if (fail == null) success(from, Ranges.of(range)); - else fail(from, Ranges.of(range), fail); + if (fail == null) + { + logger.info("Reporting success of plan {} for bootstrap of {} from {}", planId, range, from); + success(from, Ranges.of(range)); + } + else + { + logger.info("Reporting failure of plan {} for bootstrap of {} from {}", planId, range, from, fail); + fail(from, Ranges.of(range), fail); + } }, ((AccordCommandStore) commandStore()).taskExecutor()); } } @@ -302,6 +315,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamPlan plan = new StreamPlan(StreamOperation.BOOTSTRAP, 1, false, null, PreviewKind.NONE).flushBeforeTransfer(true); + logger.info("StreamPlan {} created for bootstrap of {} to {}", plan.planId(), range, to); RangesAtEndpoint ranges = RangesAtEndpoint.toDummyList(Collections.singleton(range.toKeyspaceRange())); plan.transferRanges(to, table.keyspace, ranges, table.name); StreamResultFuture future = plan.execute(); @@ -382,6 +396,12 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements super(node, node.someSequentialExecutor(), ranges, syncPoint, fetchRanges, commandStore); } + @Override + public AbstractTracker tracker() + { + return null; + } + @Override public void start() { @@ -424,10 +444,12 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements streamData.streams.forEach((range, streamInfo) -> { if (streamInfo.hasData) { + logger.info("StreamPlan {} created for bootstrap of {} from {}", streamInfo.planId, range, from); stream(streamInfo.planId).rangeReceived(range, from); } else { + logger.info("StreamPlan {} created for bootstrap of {} from {} with no data to stream; succeeding immediately.", streamInfo.planId, range, from); // if there was no data to stream, no connection is initiated, and we aren't notified via the stream // listener, so the stream initiator notifies us and we mark it complete here success(from, Ranges.of(range)); diff --git a/src/java/org/apache/cassandra/service/accord/AccordJournal.java b/src/java/org/apache/cassandra/service/accord/AccordJournal.java index 54f9522124..6633a65618 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordJournal.java +++ b/src/java/org/apache/cassandra/service/accord/AccordJournal.java @@ -50,7 +50,6 @@ import accord.local.Node; import accord.local.RedundantBefore; import accord.primitives.EpochSupplier; import accord.primitives.PartialDeps; -import accord.primitives.PartialTxn; import accord.primitives.Ranges; import accord.primitives.Route; import accord.primitives.SaveStatus; @@ -652,7 +651,7 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier if (segments != null && route != null) { for (long segment : segments) - journalTable.safeNotify(index -> index.update(segment, key.commandStoreId, txnId, (Route)route)); + journalTable.safeNotify(index -> index.update(segment, key.commandStoreId, txnId, (Route) route)); } return null; }).begin((success, fail) -> { @@ -1069,8 +1068,7 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier break; case PARTIAL_TXN: Invariants.require(partialTxn != null, "%s", this); - if (partialTxn instanceof ByteBuffer) out.write(((ByteBuffer) partialTxn).duplicate()); - else CommandSerializers.partialTxn.serialize((PartialTxn) partialTxn, out, userVersion); + CommandSerializers.partialTxn.serialize(partialTxn, out, userVersion); break; case PARTIAL_DEPS: Invariants.require(partialDeps != null, "%s", this); diff --git a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java index 40305006cd..ab4d9c98f6 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java +++ b/src/java/org/apache/cassandra/service/accord/AccordMessageSink.java @@ -163,7 +163,7 @@ public class AccordMessageSink implements MessageSink this.callbacks = callbacks; } - public AccordMessageSink(AccordAgent agent, AccordConfigurationService endpointMapper, RequestCallbacks callbacks) + public AccordMessageSink(AccordAgent agent, AccordEndpointMapper endpointMapper, RequestCallbacks callbacks) { this(agent, MessagingService.instance(), endpointMapper, callbacks); } @@ -174,7 +174,9 @@ public class AccordMessageSink implements MessageSink Verb verb = VerbMapping.getVerb(request); Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type()); Message message = Message.out(verb, request); - InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); + InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to, message); + if (endpoint == null) + return; logger.trace("Sending {} {} to {}", verb, message.payload, endpoint); messaging.send(message, endpoint); } @@ -213,7 +215,13 @@ public class AccordMessageSink implements MessageSink } Message message = Message.out(verb, request, expiresAtNanos); - InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to); + InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to, message); + if (endpoint == null) + { + executor.execute(() -> callback.onFailure(to, null)); + return null; + } + logger.trace("Sending {} {} to {}", verb, message.payload, endpoint); Cancellable cancellable = callbacks.registerAt(message.id(), executor, callback, to, nowNanos, slowAtNanos, expiresAtNanos, NANOSECONDS); messaging.send(message, endpoint); @@ -221,26 +229,31 @@ public class AccordMessageSink implements MessageSink } @Override - public void reply(Node.Id replyingToNode, ReplyContext replyContext, Reply reply) + public void reply(Node.Id replyingTo, ReplyContext replyContext, Reply reply) { ResponseContext respondTo = (ResponseContext) replyContext; - Message responseMsg = Message.responseWith(reply, respondTo); + Message message = Message.responseWith(reply, respondTo); if (!reply.isFinal()) - responseMsg = responseMsg.withFlag(MessageFlag.NOT_FINAL); + message = message.withFlag(MessageFlag.NOT_FINAL); checkReplyType(reply, respondTo); - InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode); - logger.trace("Replying {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint); - messaging.send(responseMsg, endpoint); + InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(replyingTo, message); + if (endpoint == null) + return; + + logger.trace("Replying {} {} to {}", message.verb(), message.payload, endpoint); + messaging.send(message, endpoint); } @Override - public void replyWithUnknownFailure(Node.Id replyingToNode, ReplyContext replyContext, Throwable failure) + public void replyWithUnknownFailure(Node.Id replyingTo, ReplyContext replyContext, Throwable failure) { ResponseContext respondTo = (ResponseContext) replyContext; - Message responseMsg = Message.failureResponse(RequestFailureReason.UNKNOWN, failure, respondTo); - InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode); - logger.trace("Replying with failure {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint); - messaging.send(responseMsg, endpoint); + Message message = Message.failureResponse(RequestFailureReason.UNKNOWN, failure, respondTo); + InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(replyingTo, message); + if (endpoint == null) + return; + logger.trace("Replying with failure {} {} to {}", message.verb(), message.payload, endpoint); + messaging.send(message, endpoint); } private static void checkReplyType(Reply reply, ResponseContext respondTo) diff --git a/src/java/org/apache/cassandra/service/accord/AccordOperations.java b/src/java/org/apache/cassandra/service/accord/AccordOperations.java index e7820919e1..4c0c2a7f4f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordOperations.java +++ b/src/java/org/apache/cassandra/service/accord/AccordOperations.java @@ -27,6 +27,7 @@ import java.util.stream.Collectors; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.transformations.AccordMarkHardRemoved; import org.apache.cassandra.tcm.transformations.AccordMarkStale; import org.apache.cassandra.tcm.transformations.AccordMarkRejoining; import org.apache.cassandra.utils.MBeanWrapper; @@ -55,7 +56,7 @@ public class AccordOperations implements AccordOperationsMBean ClusterMetadata metadata = ClusterMetadata.current(); info.put("EPOCH", Long.toString(metadata.epoch.getEpoch())); - String staleReplicas = metadata.accordStaleReplicas.ids().stream().sorted().map(Object::toString).collect(Collectors.joining(",")); + String staleReplicas = metadata.accordStaleReplicas.stale().stream().sorted().map(Object::toString).collect(Collectors.joining(",")); info.put("STALE_REPLICAS", staleReplicas); return info; } @@ -63,14 +64,44 @@ public class AccordOperations implements AccordOperationsMBean @Override public void accordMarkStale(List nodeIdStrings) { - Set nodeIds = nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet()); - cms.commit(new AccordMarkStale(nodeIds)); + accordMarkStale(parse(nodeIdStrings)); + } + + @Override + public void accordMarkHardRemoved(List nodeIdStrings) + { + accordMarkHardRemoved(parse(nodeIdStrings), false); + } + + @Override + public void accordForceMarkHardRemoved(List nodeIdStrings) + { + accordMarkHardRemoved(parse(nodeIdStrings), true); } @Override public void accordMarkRejoining(List nodeIdStrings) { - Set nodeIds = nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet()); + accordMarkRejoining(parse(nodeIdStrings)); + } + + private static Set parse(List nodeIdStrings) + { + return nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet()); + } + + public void accordMarkStale(Set nodeIds) + { + cms.commit(new AccordMarkStale(nodeIds)); + } + + public void accordMarkHardRemoved(Set nodeIds, boolean force) + { + cms.commit(new AccordMarkHardRemoved(nodeIds, force)); + } + + public void accordMarkRejoining(Set nodeIds) + { cms.commit(new AccordMarkRejoining(nodeIds)); } } diff --git a/src/java/org/apache/cassandra/service/accord/AccordOperationsMBean.java b/src/java/org/apache/cassandra/service/accord/AccordOperationsMBean.java index e0b0884733..033ce1ac3d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordOperationsMBean.java +++ b/src/java/org/apache/cassandra/service/accord/AccordOperationsMBean.java @@ -27,5 +27,9 @@ public interface AccordOperationsMBean void accordMarkStale(List nodeIds); + void accordMarkHardRemoved(List nodeIds); + + void accordForceMarkHardRemoved(List nodeIds); + void accordMarkRejoining(List nodeIds); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java b/src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java index 6c36d4a1fd..e75a74e1f3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java +++ b/src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java @@ -60,7 +60,13 @@ class AccordResponseVerbHandler implements IVerbHandler return; } - Node.Id from = endpointMapper.mappedId(message.from()); + Node.Id from = endpointMapper.mappedIdOrNull(message.from(), message); + if (from == null) + { + dropping.debug(message.verb(), message.from()); + return; + } + logger.trace("Receiving {} from {}", message.payload, message.from()); if (message.isFailureResponse()) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordResult.java b/src/java/org/apache/cassandra/service/accord/AccordResult.java index 8a59ba4304..e0500c48fa 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordResult.java +++ b/src/java/org/apache/cassandra/service/accord/AccordResult.java @@ -172,6 +172,10 @@ public class AccordResult extends AsyncFuture implements BiConsumer {}; AccordService i = (AccordService) instance(); - return i.configService().watermarkCollector.handler; + return i.configService().watermarkCollector().handler; } public static IVerbHandler requestHandlerOrNoop() @@ -278,7 +278,7 @@ public class AccordService implements IAccordService, Shutdownable { if (!DatabaseDescriptor.getAccordTransactionsEnabled()) { - instance = NOOP_SERVICE; + unsafeInstance = instance = NOOP_SERVICE; return null; } @@ -286,8 +286,17 @@ public class AccordService implements IAccordService, Shutdownable return (AccordService) instance; AccordService as = new AccordService(AccordTopology.tcmIdToAccord(tcmId)); - as.startup(); - replayJournal(as); + unsafeInstance = as; + as.node.unsafeSetReplaying(true); + try + { + as.startup(); + replayJournal(as); + } + finally + { + as.node.unsafeSetReplaying(false); + } as.finishInitialization(); @@ -320,23 +329,15 @@ public class AccordService implements IAccordService, Shutdownable { logger.info("Starting journal replay."); long before = Clock.Global.nanoTime(); - CommandsForKey.disableLinearizabilityViolationsReporting(); - try - { - if (as.journalConfiguration().replayMode() == RESET) - AccordKeyspace.truncateCommandsForKey(); + if (as.journalConfiguration().replayMode() == RESET) + AccordKeyspace.truncateCommandsForKey(); - as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop()); - as.journal().replay(as.node().commandStores()); - logger.info("Waiting for command stores to quiesce."); - ((AccordCommandStores)as.node.commandStores()).waitForQuiescense(); - as.journal.unsafeSetStarted(); - as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); - } - finally - { - CommandsForKey.enableLinearizabilityViolationsReporting(); - } + as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop()); + as.journal().replay(as.node().commandStores()); + logger.info("Waiting for command stores to quiesce."); + ((AccordCommandStores)as.node.commandStores()).waitForQuiescence(); + as.journal.unsafeSetStarted(); + as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start()); long after = Clock.Global.nanoTime(); logger.info("Finished journal replay. {}ms elapsed", NANOSECONDS.toMillis(after - before)); @@ -351,13 +352,18 @@ public class AccordService implements IAccordService, Shutdownable public static IAccordService instance() { - if (!DatabaseDescriptor.getAccordTransactionsEnabled()) - return NOOP_SERVICE; IAccordService i = instance; Invariants.require(i != null, "AccordService was not started"); return i; } + public static IAccordService unsafeInstance() + { + IAccordService i = unsafeInstance; + Invariants.require(i != null, "AccordService was not started"); + return i; + } + public static boolean started() { if (!DatabaseDescriptor.getAccordTransactionsEnabled()) @@ -379,7 +385,7 @@ public class AccordService implements IAccordService, Shutdownable this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal); this.configService = new AccordConfigurationService(localId, agent); this.fastPathCoordinator = AccordFastPathCoordinator.create(localId, configService); - this.messageSink = new AccordMessageSink(agent, configService, callbacks); + this.messageSink = new AccordMessageSink(agent, configService.endpointMapper(), callbacks); this.node = new Node(localId, messageSink, configService, @@ -390,18 +396,18 @@ public class AccordService implements IAccordService, Shutdownable new DefaultRandom(), scheduler, CompositeTopologySorter.create(SizeOfIntersectionSorter.SUPPLIER, - new AccordTopologySorter.Supplier(configService, DatabaseDescriptor.getNodeProximity())), + new AccordTopologySorter.Supplier(configService.endpointMapper(), DatabaseDescriptor.getNodeProximity())), DefaultRemoteListeners::new, ignore -> callbacks, DefaultProgressLogs::new, DefaultLocalListeners.Factory::new, AccordCommandStores.factory(), - new AccordInteropFactory(configService), + new AccordInteropFactory(configService.endpointMapper()), journal.durableBeforePersister(), journal); this.nodeShutdown = toShutdownable(node); - this.requestHandler = new AccordVerbHandler<>(node, configService); - this.responseHandler = new AccordResponseVerbHandler<>(callbacks, configService); + this.requestHandler = new AccordVerbHandler<>(node, configService.endpointMapper()); + this.responseHandler = new AccordResponseVerbHandler<>(callbacks, configService.endpointMapper()); } @Override @@ -416,34 +422,21 @@ public class AccordService implements IAccordService, Shutdownable configService.updateMapping(metadata); List images = journal.replayTopologies(); - - // Instantiate latest topology from the log, if known if (!images.isEmpty()) - node.commandStores().initializeTopologyUnsafe(images.get(images.size() - 1)); - - // Replay local epochs - for (TopologyUpdate image : images) - configService.reportTopology(image.global); - - // Subscribe to TCM events - ChangeListener prevListener = MetadataChangeListener.instance.collector.getAndSet(new ChangeListener() { - @Override - public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) + // Initialise command stores using latest topology from the log; + // if there are no local command stores, don't report any topologies and simply fetch the latest known in the cluster + // this avoids a registered (not joined) node learning of topologies, then later restarting with some intervening + // epochs having been garbage collected by the other nodes in the cluster + TopologyUpdate last = images.get(images.size() - 1); + if (!last.commandStores.isEmpty()) { - if (state != State.SHUTDOWN) - configService.maybeReportMetadata(next); + node.commandStores().initializeTopologyUnsafe(last); + + // Replay local epochs + for (TopologyUpdate image : images) + configService.reportTopology(image.global); } - }); - - Invariants.require((prevListener instanceof MetadataChangeListener.PreInitStateCollector), - "Listener should have been initialized with Accord pre-init state collector, but was " + prevListener.getClass()); - - MetadataChangeListener.PreInitStateCollector preinit = (MetadataChangeListener.PreInitStateCollector) prevListener; - for (ClusterMetadata item : preinit.getItems()) - { - if (item.epoch.getEpoch() > Epoch.FIRST.getEpoch()) - configService.maybeReportMetadata(item); } } @@ -464,7 +457,32 @@ public class AccordService implements IAccordService, Shutdownable TopologyRange remote = fetchTopologies(highestKnown + 1); if (remote != null) // TODO (required): if remote.min > highestKnown + 1, should we decide if we need to truncate our local topologies? Probably not until startup has finished. + { remote.forEach(configService::reportTopology, remote.min, Integer.MAX_VALUE); + if (remote.current > highestKnown) + highestKnown = remote.current; + } + + // Subscribe to TCM events, and collect any we may have missed to report now + ChangeListener prevListener = MetadataChangeListener.instance.collector.getAndSet(new ChangeListener() + { + @Override + public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) + { + if (state != State.SHUTDOWN) + configService.maybeReportMetadata(next); + } + }); + + Invariants.require((prevListener instanceof MetadataChangeListener.PreInitStateCollector), + "Listener should have been initialized with Accord pre-init state collector, but was " + prevListener.getClass()); + + MetadataChangeListener.PreInitStateCollector preinit = (MetadataChangeListener.PreInitStateCollector) prevListener; + for (ClusterMetadata item : preinit.getItems()) + { + if (item.epoch.getEpoch() > highestKnown) + configService.maybeReportMetadata(item); + } } catch (InterruptedException e) { diff --git a/src/java/org/apache/cassandra/service/accord/AccordStaleReplicas.java b/src/java/org/apache/cassandra/service/accord/AccordStaleReplicas.java index 9d0a01bad1..006b5fb0e3 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordStaleReplicas.java +++ b/src/java/org/apache/cassandra/service/accord/AccordStaleReplicas.java @@ -20,13 +20,12 @@ package org.apache.cassandra.service.accord; import java.io.IOException; import java.util.Objects; -import java.util.Set; import javax.annotation.concurrent.Immutable; -import com.google.common.collect.ImmutableSet; - import accord.local.Node; +import accord.utils.SortedArrays.SortedArrayList; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.serializers.TopologySerializers; @@ -36,24 +35,28 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.CollectionSerializers; +import static accord.topology.Topology.NO_IDS; + @Immutable public class AccordStaleReplicas implements MetadataValue { - public static final AccordStaleReplicas EMPTY = new AccordStaleReplicas(ImmutableSet.of(), Epoch.EMPTY); + public static final AccordStaleReplicas EMPTY = new AccordStaleReplicas(NO_IDS, NO_IDS, Epoch.EMPTY); - private final Set staleIds; + private final SortedArrayList stale; + private final SortedArrayList hardRemoved; private final Epoch lastModified; - public AccordStaleReplicas(Set staleIds, Epoch lastModified) + public AccordStaleReplicas(SortedArrayList stale, SortedArrayList hardRemoved, Epoch lastModified) { - this.staleIds = staleIds; + this.stale = stale; + this.hardRemoved = hardRemoved; this.lastModified = lastModified; } @Override public AccordStaleReplicas withLastModified(Epoch epoch) { - return new AccordStaleReplicas(staleIds, epoch); + return new AccordStaleReplicas(stale, hardRemoved, epoch); } @Override @@ -62,38 +65,36 @@ public class AccordStaleReplicas implements MetadataValue return lastModified; } - public AccordStaleReplicas withNodeIds(Set ids) + public AccordStaleReplicas withStale(SortedArrayList stale) { - ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); - Set newIds = builder.addAll(staleIds).addAll(ids).build(); - return new AccordStaleReplicas(newIds, lastModified); + return new AccordStaleReplicas(this.stale.with(stale), hardRemoved, lastModified); } - public AccordStaleReplicas without(Set ids) + public AccordStaleReplicas withHardRemoved(SortedArrayList hardRemoved) { - ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); - - for (Node.Id staleId : staleIds) - if (!ids.contains(staleId)) - builder.add(staleId); - - return new AccordStaleReplicas(builder.build(), lastModified); + return new AccordStaleReplicas(stale, this.hardRemoved.with(hardRemoved), lastModified); } - public boolean contains(Node.Id nodeId) + public AccordStaleReplicas withoutStale(SortedArrayList withoutStale) { - return staleIds.contains(nodeId); + + return new AccordStaleReplicas(this.stale.without(withoutStale), hardRemoved, lastModified); } - public Set ids() + public SortedArrayList stale() { - return staleIds; + return stale; + } + + public SortedArrayList hardRemoved() + { + return hardRemoved; } @Override public String toString() { - return "AccordStaleReplicas{staleIds=" + staleIds + ", lastModified=" + lastModified + '}'; + return "AccordStaleReplicas{staleIds=" + stale + ", hardRemoved=" + hardRemoved + ", lastModified=" + lastModified + '}'; } @Override @@ -102,36 +103,64 @@ public class AccordStaleReplicas implements MetadataValue if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AccordStaleReplicas that = (AccordStaleReplicas) o; - return Objects.equals(staleIds, that.staleIds) && Objects.equals(lastModified, that.lastModified); + return Objects.equals(stale, that.stale) && Objects.equals(lastModified, that.lastModified); } @Override public int hashCode() { - return Objects.hash(staleIds, lastModified); + return Objects.hash(stale, lastModified); } public static final MetadataSerializer serializer = new MetadataSerializer<>() { + private static final int HAS_STALE_IDS = 0x1; + private static final int HAS_HARD_REMOVED_IDS = 0x2; + @Override public void serialize(AccordStaleReplicas replicas, DataOutputPlus out, Version version) throws IOException { - CollectionSerializers.serializeCollection(replicas.staleIds, out, TopologySerializers.nodeId); + int flags = 0; + if (!replicas.stale.isEmpty()) + flags |= HAS_STALE_IDS; + if (!replicas.hardRemoved.isEmpty()) + flags |= HAS_HARD_REMOVED_IDS; + out.writeUnsignedVInt32(flags); + if (!replicas.stale.isEmpty()) + CollectionSerializers.serializeCollection(replicas.stale, out, TopologySerializers.nodeId); + if (!replicas.hardRemoved.isEmpty()) + CollectionSerializers.serializeCollection(replicas.hardRemoved, out, TopologySerializers.nodeId); Epoch.serializer.serialize(replicas.lastModified, out, version); } @Override public AccordStaleReplicas deserialize(DataInputPlus in, Version version) throws IOException { - return new AccordStaleReplicas(CollectionSerializers.deserializeSet(in, TopologySerializers.nodeId), - Epoch.serializer.deserialize(in, version)); + int flags = in.readUnsignedVInt32(); + SortedArrayList stale = NO_IDS, hardRemoved = NO_IDS; + if ((flags & HAS_STALE_IDS) != 0) + stale = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + if ((flags & HAS_HARD_REMOVED_IDS) != 0) + hardRemoved = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + Epoch lastModified = Epoch.serializer.deserialize(in, version); + return new AccordStaleReplicas(stale, hardRemoved, lastModified); } @Override public long serializedSize(AccordStaleReplicas replicas, Version version) { - return CollectionSerializers.serializedCollectionSize(replicas.staleIds, TopologySerializers.nodeId) - + Epoch.serializer.serializedSize(replicas.lastModified, version); + int flags = 0; + if (!replicas.stale.isEmpty()) + flags |= HAS_STALE_IDS; + if (!replicas.hardRemoved.isEmpty()) + flags |= HAS_HARD_REMOVED_IDS; + long size = TypeSizes.sizeofUnsignedVInt(flags); + if (!replicas.stale.isEmpty()) + size += CollectionSerializers.serializedCollectionSize(replicas.stale, TopologySerializers.nodeId); + if (!replicas.hardRemoved.isEmpty()) + size += CollectionSerializers.serializedCollectionSize(replicas.hardRemoved, TopologySerializers.nodeId); + size += Epoch.serializer.serializedSize(replicas.lastModified, version); + return size; } }; } diff --git a/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java b/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java index c8b5e75685..8a1c296b61 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java +++ b/src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -36,13 +34,14 @@ import accord.local.Node; import accord.messages.SimpleReply; import accord.primitives.Ranges; import accord.utils.Invariants; +import accord.utils.SortedList; +import accord.utils.SortedListSet; +import accord.utils.UnhandledEnum; import org.agrona.collections.Int2ObjectHashMap; import org.agrona.collections.Long2ObjectHashMap; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.exceptions.RequestFailure; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.io.UnversionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -54,7 +53,6 @@ import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; import org.apache.cassandra.service.accord.serializers.KeySerializers; import org.apache.cassandra.service.accord.serializers.TopologySerializers; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.CollectionSerializers; import org.apache.cassandra.utils.NoSpamLogger; @@ -79,7 +77,7 @@ public class AccordSyncPropagator interface Listener { void onEndpointAck(Node.Id id, long epoch); - void onComplete(long epoch); + void onEndpointNack(Node.Id id, long epoch); } private interface ReportPending @@ -191,19 +189,17 @@ public class AccordSyncPropagator private final Node.Id localId; private final AccordEndpointMapper endpointMapper; private final MessageDelivery messagingService; - private final IFailureDetector failureDetector; private final ScheduledExecutorPlus scheduler; private final Listener listener; private final ConcurrentHashMap retryingNotifications = new ConcurrentHashMap<>(); public AccordSyncPropagator(Node.Id localId, AccordEndpointMapper endpointMapper, - MessageDelivery messagingService, IFailureDetector failureDetector, ScheduledExecutorPlus scheduler, + MessageDelivery messagingService, ScheduledExecutorPlus scheduler, Listener listener) { this.localId = localId; this.endpointMapper = endpointMapper; this.messagingService = messagingService; - this.failureDetector = failureDetector; this.scheduler = scheduler; this.listener = listener; } @@ -234,20 +230,17 @@ public class AccordSyncPropagator public void onNodesRemoved(Node.Id removed) { long[] toAck; - boolean[] syncCompletedFor; synchronized (AccordSyncPropagator.this) { PendingEpochs pendingEpochs = pending.remove(removed.id); if (pendingEpochs == null) return; toAck = new long[pendingEpochs.size()]; - syncCompletedFor = new boolean[pendingEpochs.size()]; Long2ObjectHashMap.KeyIterator it = pendingEpochs.keySet().iterator(); for (int i = 0; it.hasNext(); i++) { long epoch = it.nextLong(); toAck[i] = epoch; - syncCompletedFor[i] = hasSyncCompletedFor(epoch); } Arrays.sort(toAck); } @@ -256,19 +249,15 @@ public class AccordSyncPropagator { long epoch = toAck[i]; listener.onEndpointAck(removed, epoch); - if (syncCompletedFor[i]) - listener.onComplete(epoch); } } - public void reportSyncComplete(long epoch, Collection notify, Node.Id syncCompleteId) + public void reportCoordinationReady(long epoch, SortedList notify, Node.Id syncCompleteId) { - if (notify.isEmpty()) - { - listener.onComplete(epoch); - return; - } - report(epoch, notify, PendingEpoch::syncComplete, syncCompleteId); + SortedListSet remaining = SortedListSet.allOf(notify); + if (remaining.remove(localId)) + listener.onEndpointAck(localId, epoch); + report(epoch, remaining, PendingEpoch::syncComplete, syncCompleteId); } public void reportClosed(long epoch, Collection notify, Ranges closed) @@ -281,31 +270,25 @@ public class AccordSyncPropagator report(epoch, notify, PendingEpoch::retired, retired); } - private synchronized void report(long epoch, Collection notify, ReportPending report, T param) + private void report(long epoch, Collection notify, ReportPending report, T param) { // TODO (efficiency, now): for larger clusters this can be a problem as we trigger 1 msg for each instance, so in a 1k cluster its 1k messages; this can cause a thundering herd problem // this is mostly a problem for reportSyncComplete as we include every node in the cluster, for reportClosed/reportRetired these tend to use only the nodes that are replicas of the range, // and there is currently an assumption that sub-ranges are done, so only impacting a handful of nodes. // TODO (correctness, now): during a host replacement multiple epochs are generated (move the range, remove the node), so its possible that notify will never be able to send the notification as the node is leaving the cluster notify.forEach(id -> { - PendingEpoch pendingEpoch = pending.computeIfAbsent(id.id, ignore -> new PendingEpochs()) - .computeIfAbsent(epoch, PendingEpoch::new); - Notification notification = report.report(pendingEpoch, param); + Notification notification; + synchronized (this) + { + PendingEpoch pendingEpoch = pending.computeIfAbsent(id.id, ignore -> new PendingEpochs()) + .computeIfAbsent(epoch, PendingEpoch::new); + notification = report.report(pendingEpoch, param); + } if (notification != null) notify(id, notification); }); } - private boolean hasSyncCompletedFor(long epoch) - { - return pending.values().stream().noneMatch(node -> { - PendingEpoch pending = node.get(epoch); - if (pending == null) - return false; - return !pending.syncComplete.isEmpty(); - }); - } - private void scheduleRetry(Node.Id to, Notification notification) { Notification retry = new Notification(notification.epoch, notification.syncComplete, notification.closed, notification.retired, notification.attempts + 1); @@ -329,65 +312,66 @@ public class AccordSyncPropagator private boolean notify(Node.Id to, Notification notification) { - InetAddressAndPort toEp = endpointMapper.mappedEndpoint(to); - Message msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, notification); - RequestCallback cb = new RequestCallback<>() - { - @Override - public void onResponse(Message msg) - { - Invariants.require(msg.payload == SimpleReply.Ok, "Unexpected message: %s", msg); - Set completedEpochs = new HashSet<>(); - synchronized (AccordSyncPropagator.this) - { - pending.ack(to, notification); - long epoch = notification.epoch; - if (notification.syncComplete.contains(localId)) - { - if (hasSyncCompletedFor(epoch)) - completedEpochs.add(epoch); - } - } - - long epoch = notification.epoch; - listener.onEndpointAck(to, epoch); - if (completedEpochs.contains(epoch)) - listener.onComplete(epoch); - } - - @Override - public void onFailure(InetAddressAndPort from, RequestFailure failure) - { - scheduleRetry(to, notification); - } - - @Override - public boolean invokeOnFailure() - { - return true; - } - }; - if (!failureDetector.isAlive(toEp)) - { - // was the endpoint removed from membership? - ClusterMetadata metadata = ClusterMetadata.current(); - if (Gossiper.instance.getEndpointStateForEndpoint(toEp) == null && !metadata.directory.allJoinedEndpoints().contains(toEp) && !metadata.fullCMSMembers().contains(toEp)) - { - // endpoint no longer exists... - cb.onResponse(msg.responseWith(SimpleReply.Ok)); - return true; - } - noSpamLogger.warn("Node{} is not alive, unable to notify of {}", to, notification); - scheduleRetry(to, notification); + InetAddressAndPort toEp = endpointMapper.mappedEndpointOrNull(to, notification); + if (toEp == null) return false; + + // was the endpoint removed from membership? + AccordEndpointMapper.NodeStatus nodeStatus = endpointMapper.nodeStatus(to); + switch (nodeStatus) + { + default: throw new UnhandledEnum(nodeStatus); + case UNHEALTHY: + if (!endpointMapper.isRemoved(to)) + { + noSpamLogger.warn("Node{} is not alive, unable to notify of {}", to, notification); + scheduleRetry(to, notification); + return false; + } + // fall through to UNKNOWN, as we have been removed from the cluster in the latest epoch + case UNKNOWN: + // endpoint is not a member of the latest epoch + pending.ack(to, notification); + listener.onEndpointNack(to, notification.epoch); + return true; + + case HEALTHY: + Message msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, notification); + RequestCallback cb = new RequestCallback<>() + { + @Override + public void onResponse(Message msg) + { + Invariants.require(msg.payload == SimpleReply.Ok, "Unexpected message: %s", msg); + synchronized (AccordSyncPropagator.this) + { + pending.ack(to, notification); + } + + long epoch = notification.epoch; + listener.onEndpointAck(to, epoch); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailure failure) + { + scheduleRetry(to, notification); + } + + @Override + public boolean invokeOnFailure() + { + return true; + } + }; + messagingService.sendWithCallback(msg, toEp, cb); + return true; } - messagingService.sendWithCallback(msg, toEp, cb); - return true; } public static class Notification { - public static final UnversionedSerializer serializer = new UnversionedSerializer() + public static final UnversionedSerializer serializer = new UnversionedSerializer<>() { @Override public void serialize(Notification notification, DataOutputPlus out) throws IOException diff --git a/src/java/org/apache/cassandra/service/accord/AccordTopology.java b/src/java/org/apache/cassandra/service/accord/AccordTopology.java index 8c60ecae5e..db18ba4c9d 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTopology.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTopology.java @@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -80,9 +79,9 @@ public class AccordTopology return new Id(nodeId.id()); } - private static class ShardLookup extends HashMap + public static class ShardLookup extends HashMap { - private Shard createOrReuse(TinyEnumSet flags, accord.primitives.Range range, SortedArrayList nodes, SortedArrayList fastPath, Set joining) + private Shard createOrReuse(TinyEnumSet flags, accord.primitives.Range range, SortedArrayList nodes, SortedArrayList fastPath, SortedArrayList hardRemoved) { Shard prev = get(range); if (prev != null @@ -90,10 +89,10 @@ public class AccordTopology && prev.nodes.equals(nodes) && prev.fastPathElectorateSize == fastPath.size() && prev.nodes.without(prev.notInFastPath).equals(fastPath) - && joining.size() == prev.joining.size() && prev.joining.containsAll(joining)) + && hardRemoved.size() == prev.hardRemoved.size() && prev.hardRemoved.containsAll(hardRemoved)) return prev; - return Shard.create(range, nodes, fastPath, joining, flags); + return Shard.create(range, nodes, fastPath, hardRemoved, flags); } } @@ -102,14 +101,12 @@ public class AccordTopology private final KeyspaceMetadata keyspace; private final List> ranges; private final SortedArrayList nodes; - private final Set pending; - private KeyspaceShard(KeyspaceMetadata keyspace, List> ranges, SortedArrayList nodes, Set pending) + private KeyspaceShard(KeyspaceMetadata keyspace, List> ranges, SortedArrayList nodes) { this.keyspace = keyspace; this.ranges = ranges; this.nodes = nodes; - this.pending = pending; } // return the keyspace fast path strategy if the inherit keyspace strategy is used @@ -122,7 +119,7 @@ public class AccordTopology return strategy; } - List createForTable(Epoch epoch, TableMetadata metadata, Set unavailable, Map dcMap, ShardLookup lookup) + List createForTable(Epoch epoch, TableMetadata metadata, SortedArrayList unavailable, SortedArrayList hardRemoved, Map dcMap, ShardLookup lookup) { Ranges ranges = this.ranges.stream() .map(range -> Ranges.single(AccordTopology.range(metadata.id, range))) @@ -139,7 +136,7 @@ public class AccordTopology flags = flags.with(Shard.Flag.PENDING_REMOVAL); if (metadata.epoch.isEqualOrAfter(epoch)) flags = flags.with(Shard.Flag.MUST_WITNESS); - shards.add(lookup.createOrReuse(flags, range, nodes, electorate, pending)); + shards.add(lookup.createOrReuse(flags, range, nodes, electorate, hardRemoved.intersecting(nodes))); } return shards; } @@ -159,15 +156,7 @@ public class AccordTopology .map(AccordTopology::tcmIdToAccord) .sorted().toArray(Id[]::new)); - Set pending = readEndpoints.equals(writeEndpoints) ? - Collections.emptySet() : - writeEndpoints.stream() - .filter(e -> !readEndpoints.contains(e)) - .map(directory::peerId) - .map(AccordTopology::tcmIdToAccord) - .collect(Collectors.toSet()); - - return new KeyspaceShard(keyspace, ranges, nodes, pending); + return new KeyspaceShard(keyspace, ranges, nodes); } public static List forKeyspace(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory) @@ -217,7 +206,7 @@ public class AccordTopology return shards; } - public List nodes() + public SortedArrayList nodes() { return nodes; } @@ -296,7 +285,7 @@ public class AccordTopology AccordStaleReplicas staleReplicas) { List res = new ArrayList<>(); - Set unavailable = accordFastPath.unavailableIds(); + SortedArrayList unavailable = accordFastPath.unavailableIds(); Map dcMap = createDCMap(directory); for (KeyspaceMetadata keyspace : schema.getKeyspaces()) @@ -305,16 +294,17 @@ public class AccordTopology if (tables.isEmpty()) continue; List ksShards = KeyspaceShard.forKeyspace(keyspace, placements, directory); - tables.forEach(table -> ksShards.forEach(shard -> res.addAll(shard.createForTable(epoch, table, unavailable, dcMap, lookup)))); + tables.forEach(table -> ksShards.forEach(shard -> res.addAll(shard.createForTable(epoch, table, unavailable, staleReplicas.hardRemoved(), dcMap, lookup)))); } res.sort((a, b) -> a.range.compare(b.range)); - List removed = directory.removedNodes().stream() - .filter(n -> n.removedIn.equals(epoch)) - .map(n -> tcmIdToAccord(n.id)) - .collect(Collectors.toList()); - - return new Topology(epoch.getEpoch(), SortedArrayList.copySorted(removed, Id[]::new), SortedArrayList.copyUnsorted(staleReplicas.ids(), Id[]::new), res.toArray(new Shard[0])); + SortedArrayList removed = SortedArrayList.ofUnsorted( + directory.removedNodes().stream() + .filter(n -> n.removedIn.equals(epoch)) + .map(n -> tcmIdToAccord(n.id)) + .toArray(Id[]::new) + ); + return new Topology(epoch.getEpoch(), removed, staleReplicas.hardRemoved(), staleReplicas.stale(), res.toArray(new Shard[0])); } public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup) diff --git a/src/java/org/apache/cassandra/service/accord/AccordTracing.java b/src/java/org/apache/cassandra/service/accord/AccordTracing.java index 8bbae998ca..ff8bddfe8f 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordTracing.java +++ b/src/java/org/apache/cassandra/service/accord/AccordTracing.java @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import accord.api.Tracing; import accord.coordinate.Coordination; import accord.coordinate.Coordination.CoordinationKind; -import accord.coordinate.tracking.AbstractTracker; import accord.local.CommandStore; import accord.local.Node; import accord.primitives.Participants; @@ -158,9 +157,12 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener void truncateInternal(int eraseBefore) { - System.arraycopy(events, eraseBefore, events, 0, size - eraseBefore); - Arrays.fill(events, size - eraseBefore, size, null); - size -= eraseBefore; + if (events != null) + { + System.arraycopy(events, eraseBefore, events, 0, size - eraseBefore); + Arrays.fill(events, size - eraseBefore, size, null); + size -= eraseBefore; + } } public TxnEvent get(int index) @@ -314,6 +316,7 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener if (!globalCount.admit()) return null; + incrementSeen(); return newTrace(kind, null); } @@ -673,9 +676,11 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener private synchronized void untrace(TxnId txnId) { txnIdMap.compute(txnId, (ignore, cur) -> { - if (cur == null || cur.owner == this) - return null; - return cur; + if (cur == null || cur.owner != this) + return cur; + + cur.stopTracing(); + return cur.eraseEvents(globalCount); }); } @@ -968,29 +973,14 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener return cur; event.trace(null, "Failed Coordination Dump"); + Coordination.traceStart(event, coordination); + SortedListMap replies = coordination.replies(); + if (replies != null) { - String description = coordination.describe(); - if (description != null) - event.trace(null, "Description: %s", description); - } - { - Participants scope = coordination.scope(); - if (scope != null) - event.trace(null, "Scope: %s", scope); - } - { - AbstractTracker tracker = coordination.tracker(); - if (tracker != null) - event.trace(null, "Tracker: %s", tracker.summariseTracker()); - } - { - SortedListMap replies = coordination.replies(); - if (replies != null) - { - for (int i = 0 ; i < replies.domainSize() ; ++i) - event.trace(null, "from %s: %s", replies.getKey(i), replies.getValue(i)); - } + for (int i = 0 ; i < replies.domainSize() ; ++i) + event.trace(null, "from %s: %s", replies.getKey(i), replies.getValue(i)); } + Coordination.traceStop(event, coordination); return cur; }); } diff --git a/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java b/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java index a909ea8070..d90b449b47 100644 --- a/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java +++ b/src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java @@ -60,7 +60,13 @@ public class AccordVerbHandler implements IVerbHandler * TODO (desired): messages are retained on heap until the node catches up to waitForEpoch, * which can be problematic in absense of proper Accord<->Messaging backpressure */ - Node.Id fromNodeId = endpointMapper.mappedId(message.from()); + Node.Id fromNodeId = endpointMapper.mappedIdOrNull(message.from(), message); + if (fromNodeId == null) + { + dropping.debug(message.verb(), message.from()); + return; + } + long waitForEpoch = request.waitForEpoch(); if (node.topology().hasAtLeastEpoch(waitForEpoch)) { diff --git a/src/java/org/apache/cassandra/service/accord/EndpointMapping.java b/src/java/org/apache/cassandra/service/accord/EndpointMapping.java index 1d5a21995a..5ee46c0047 100644 --- a/src/java/org/apache/cassandra/service/accord/EndpointMapping.java +++ b/src/java/org/apache/cassandra/service/accord/EndpointMapping.java @@ -22,17 +22,78 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; + import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import accord.local.Node; import accord.utils.Invariants; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.NoSpamLogger; + +import static java.util.concurrent.TimeUnit.MINUTES; class EndpointMapping implements AccordEndpointMapper { + static class Updateable implements AccordEndpointMapper + { + private volatile EndpointMapping mapping = EMPTY; + + @Nullable + @Override + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped) + { + return mapping.mappedIdOrNull(endpoint, logIdentityIfUnmapped); + } + + @Nullable + @Override + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped) + { + return mapping.mappedEndpointOrNull(id, logIdentityIfUnmapped); + } + + @Override + public Map removedNodes() + { + return mapping.removedNodes; + } + + @Override + public NodeStatus nodeStatus(Node.Id id) + { + return mapping.nodeStatus(id); + } + + @Override + public synchronized void updateMapping(ClusterMetadata metadata) + { + if (metadata.epoch.getEpoch() > this.mapping.epoch()) + this.mapping = AccordTopology.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory); + } + + public synchronized void updateMapping(EndpointMapping newMapping) + { + if (newMapping.epoch > mapping.epoch) + mapping = newMapping; + } + } + + + private static final Logger logger = LoggerFactory.getLogger(EndpointMapping.class); + private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, MINUTES); + public static final EndpointMapping EMPTY = new EndpointMapping(0, ImmutableBiMap.of(), ImmutableMap.of()); private final long epoch; private final ImmutableBiMap mapping; @@ -62,21 +123,53 @@ class EndpointMapping implements AccordEndpointMapper return new ArrayList<>(mapping.keySet()); } + @Override + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object logIdentityIfUnmapped) + { + Node.Id id = mapping.inverse().get(endpoint); + if (id != null) + return id; + if (logIdentityIfUnmapped == null) noSpamLogger.warn("Could not find Node.Id for endpoint {}", endpoint); + else noSpamLogger.warn("Could not find Node.Id for endpoint {} on behalf of {}", endpoint, logIdentityIfUnmapped); + return null; + } + + @Override + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object logIdentityIfUnmapped) + { + InetAddressAndPort ep = mapping.get(id); + if (ep != null) + return ep; + if (logIdentityIfUnmapped == null) noSpamLogger.warn("Could not find InetAddressAndPort for Node.Id {}", id); + else noSpamLogger.warn("Could not find InetAddressAndPort for Node.Id {} on behalf of {}", id, logIdentityIfUnmapped); + return null; + } + + @Override public Map removedNodes() { return removedNodes; } @Override - public Node.Id mappedIdOrNull(InetAddressAndPort endpoint) + public NodeStatus nodeStatus(Node.Id id) { - return mapping.inverse().get(endpoint); - } + InetAddressAndPort ep = mappedEndpointOrNull(id); + if (ep == null) + return NodeStatus.UNKNOWN; - @Override - public InetAddressAndPort mappedEndpointOrNull(Node.Id id) - { - return mapping.get(id); + EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep); + if (epState == null) + return NodeStatus.UNKNOWN; + + if (!epState.isAlive()) + return NodeStatus.UNHEALTHY; + + VersionedValue event = epState.getApplicationState(ApplicationState.SEVERITY); + if (event == null) + return NodeStatus.HEALTHY; // should we delineate this status better? + + return Double.parseDouble(event.value) == 0.0 ? NodeStatus.UNHEALTHY : NodeStatus.HEALTHY; } static class Builder diff --git a/src/java/org/apache/cassandra/service/accord/WatermarkCollector.java b/src/java/org/apache/cassandra/service/accord/WatermarkCollector.java index 64155e0533..6993504a5c 100644 --- a/src/java/org/apache/cassandra/service/accord/WatermarkCollector.java +++ b/src/java/org/apache/cassandra/service/accord/WatermarkCollector.java @@ -81,7 +81,7 @@ public class WatermarkCollector implements ConfigurationService.Listener synced = new Long2LongHashMap(-1); } - @Override public AsyncResult onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync) + @Override public AsyncResult onTopologyUpdate(Topology topology) { return null; } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java index 4f3e0ace11..eb69136d6e 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordAgent.java @@ -62,9 +62,11 @@ import accord.utils.async.AsyncChain; import accord.utils.async.AsyncChains; import accord.utils.async.Cancellable; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.exceptions.RequestTimeoutException; import org.apache.cassandra.metrics.AccordReplicaMetrics; import org.apache.cassandra.net.ResponseContext; +import org.apache.cassandra.service.RetryStrategy; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTracing; import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys; @@ -88,6 +90,7 @@ import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.fetch import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.recover; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryBootstrap; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryDurability; +import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryJoinBootstrap; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retrySyncPoint; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept; import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead; @@ -139,10 +142,48 @@ public class AccordAgent implements Agent, OwnershipEventListener } @Override - public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure) + public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure) { - logger.error("Failed bootstrap at {} for {}", phase, ranges, failure); - AccordService.instance().scheduler().once(retry, retryBootstrap.computeWait(attempts, MICROSECONDS), MICROSECONDS); + RetryStrategy strategy; + String message; + SystemKeyspace.BootstrapState bootstrapState = SystemKeyspace.getBootstrapState(); + switch (bootstrapState) + { + default: throw new UnhandledEnum(bootstrapState); + case IN_PROGRESS: + case NEEDS_BOOTSTRAP: + message = "Failed bootstrap (for joining) at {} for {}{}"; + strategy = retryJoinBootstrap; + break; + case COMPLETED: + case DECOMMISSIONED: + message = "Failed bootstrap at {} for {}{}"; + strategy = retryBootstrap; + break; + } + long retryDelayMicros = strategy.computeWait(attempts, MICROSECONDS); + if (retryDelayMicros < 0) + { + if (strategy == retryJoinBootstrap) + { + logger.error(message, phase, ranges, ". Retry strategy giving up. Not yet joined, so failing bootstrap.", failure); + fail.run(); + } + else + { + // TODO (expected): we should be able to resume these without restarting (but for now we just shouldn't configure a retry limit) + // failing would prevent the node processing all epochs (as this feeds into the epoch readiness), so we just drop in this case + logger.error(message, phase, ranges, ". Retry strategy giving up. To resume you will need to restart.", failure); + } + } + else + { + logger.error(message, phase, ranges, ". Retrying in " + retryDelayMicros + "us.", failure); + AccordService.instance().scheduler().once(() -> { + logger.info("Retrying bootstrap of {}", ranges); + retry.run(); + }, retryDelayMicros, MICROSECONDS); + } } @Override diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java b/src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java index a9f004b350..6d1c42b490 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java @@ -22,16 +22,14 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.Set; +import javax.annotation.Nullable; + import accord.api.TopologySorter; import accord.local.Node; import accord.topology.ShardSelection; import accord.topology.Topologies; import accord.topology.Topology; import accord.utils.SortedList; -import org.apache.cassandra.gms.ApplicationState; -import org.apache.cassandra.gms.EndpointState; -import org.apache.cassandra.gms.Gossiper; -import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.Endpoint; import org.apache.cassandra.locator.InetAddressAndPort; @@ -40,6 +38,8 @@ import org.apache.cassandra.service.accord.AccordEndpointMapper; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Sortable; +import static org.apache.cassandra.service.accord.AccordEndpointMapper.NodeStatus.HEALTHY; + public class AccordTopologySorter implements TopologySorter { public static class Supplier implements TopologySorter.Supplier @@ -96,54 +96,25 @@ public class AccordTopologySorter implements TopologySorter @Override public int compare(Node.Id node1, Node.Id node2, ShardSelection shards) { - return comparator.compare(() -> mapper.mappedEndpoint(node1), () -> mapper.mappedEndpoint(node2)); + InetAddressAndPort ep1 = mapper.mappedEndpointOrNull(node1); + InetAddressAndPort ep2 = mapper.mappedEndpointOrNull(node2); + return compare(comparator, ep1, ep2); } @Override public boolean isFaulty(Node.Id node) { - InetAddressAndPort ep = mapper.mappedEndpointOrNull(node); - if (ep == null) - return true; - - EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep); - if (epState == null) - return true; - - if (!epState.isAlive()) - return true; - - VersionedValue event = epState.getApplicationState(ApplicationState.SEVERITY); - if (event == null) - return false; - - return Double.parseDouble(event.value) == 0.0; + return mapper.nodeStatus(node) != HEALTHY; } - private static class EndpointTuple implements Endpoint - { - final InetAddressAndPort endpoint; - - private EndpointTuple(InetAddressAndPort endpoint) - { - this.endpoint = endpoint; - } - - @Override - public InetAddressAndPort endpoint() - { - return endpoint; - } - } - - private static class SortableEndpoints extends ArrayList implements Sortable + private static class SortableEndpoints extends ArrayList implements Sortable { public SortableEndpoints(int initialCapacity) { super(initialCapacity); } - public SortableEndpoints sorted(Comparator comparator) + public SortableEndpoints sorted(Comparator comparator) { sort(comparator); return this; @@ -152,8 +123,19 @@ public class AccordTopologySorter implements TopologySorter static SortableEndpoints from(Set nodes, AccordEndpointMapper mapper) { SortableEndpoints result = new SortableEndpoints(nodes.size()); - nodes.forEach(id -> result.add(new EndpointTuple(mapper.mappedEndpoint(id)))); + nodes.forEach(id -> { + InetAddressAndPort ep = mapper.mappedEndpointOrNull(id); + if (ep != null) + result.add(ep); + }); return result; } } + + private static int compare(Comparator comparator, @Nullable InetAddressAndPort ep1, @Nullable InetAddressAndPort ep2) + { + if (ep1 == null || ep2 == null) + return ep1 == ep2 ? 0 : ep1 == null ? 1 : -1; + return comparator.compare(ep1, ep2); + } } diff --git a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java index 6fcacee42a..74d1777aa2 100644 --- a/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java +++ b/src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java @@ -40,7 +40,7 @@ public class AccordWaitStrategies static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead; static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait; static TimeoutStrategy fetchTxn, fetchSyncPoint; - static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap; + static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap, retryJoinBootstrap; static RetryStrategy retryFetchMinEpoch, retryFetchTopology; public static @Nullable TimeoutStrategy slowRead(@Nullable TxnId txnId) @@ -98,6 +98,7 @@ public class AccordWaitStrategies setRetrySyncPoint(config.retry_syncpoint); setRetryDurability(config.retry_durability); setRetryBootstrap(config.retry_bootstrap); + setRetryJoinBootstrap(config.retry_join_bootstrap); setRetryFetchMinEpoch(config.retry_fetch_min_epoch); setRetryFetchTopology(config.retry_fetch_topology); } @@ -172,6 +173,11 @@ public class AccordWaitStrategies retryBootstrap = spec.retry(); } + public static void setRetryJoinBootstrap(StringRetryStrategy spec) + { + retryJoinBootstrap = spec.retry(); + } + public static void setRetryFetchMinEpoch(StringRetryStrategy spec) { retryFetchMinEpoch = spec.retry(); diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java index d9ea6a8024..eddd77abb6 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java @@ -66,6 +66,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -93,6 +94,7 @@ import org.apache.cassandra.transport.Dispatcher; import static accord.coordinate.CoordinationAdapter.Factory.Kind.Standard; import static accord.primitives.Txn.Kind.Write; import static accord.topology.Topologies.SelectNodeOwnership.SHARE; +import static accord.utils.Invariants.illegalState; import static accord.utils.Invariants.requireArgument; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics; import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics; @@ -190,7 +192,10 @@ public class AccordInteropExecution implements ReadCoordinator for (int i=0; i message, InetAddressAndPort to, RequestCallback callback) { - Node.Id id = endpointMapper.mappedId(to); + Node.Id id = endpointMapper.mappedIdOrNull(to, message); + if (id == null) + { + callback.onFailure(to, RequestFailure.NODE_DOWN); + return; + } // TODO (desired): It would be better to use the re-use the command from the transaction but it's fragile // to try and figure out exactly what changed for things like read repair and short read protection // Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be @@ -213,7 +223,12 @@ public class AccordInteropExecution implements ReadCoordinator { requireArgument(message.payload.potentialTxnConflicts().allowed); requireArgument(message.payload.getTableIds().size() == 1); - Node.Id id = endpointMapper.mappedId(to); + Node.Id id = endpointMapper.mappedIdOrNull(to, message); + if (id == null) + { + callback.onFailure(to, RequestFailure.NODE_DOWN); + return; + } Participants readScope = Participants.singleton(txn.read().keys().domain(), new TokenKey(message.payload.getTableIds().iterator().next(), message.payload.key().getToken())); AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), message.payload); node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this)); @@ -354,8 +369,11 @@ public class AccordInteropExecution implements ReadCoordinator private void sendStableToUncontacted() { for (Node.Id to : executeTopology.nodes()) - if (!contacted.contains(endpointMapper.mappedEndpoint(to))) + { + InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to); + if (endpoint != null && !contacted.contains(endpoint)) node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps)); + } } public void start() diff --git a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java index 3f02668070..6b6e204c63 100644 --- a/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java +++ b/src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java @@ -31,7 +31,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.RequestCallback; -import static accord.messages.ReadData.CommitOrReadNack.Insufficient; +import static accord.messages.ReadData.CommitOrReadNack.InsufficientAndWaiting; public abstract class AccordInteropReadCallback implements Callback { @@ -62,7 +62,7 @@ public abstract class AccordInteropReadCallback implements Callback including; private final Epoch minEpoch = ClusterMetadata.current().epoch; private volatile Throwable shouldAbort = null; private volatile Thread waiting; - public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection> ranges, boolean requireAllEndpoints) + public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection> ranges, boolean requireAllEndpoints, List endpoints) { this.ctx = ctx; this.cfs = cfs; this.repairId = repairId; + // TODO (desired): support unsafe configuration where we permit less than a quorum, + // but this is challenging to do safely as repair has no concept of participants from earlier epochs this.syncRemote = requireAllEndpoints ? All : Quorum; + if (endpoints != null) + { + including = new ArrayList<>(endpoints.size()); + AccordEndpointMapper mapper = AccordService.instance().configService().endpointMapper(); + for (InetAddressAndPort ep : endpoints) + { + Node.Id id = mapper.mappedIdOrNull(ep); + if (id == null) + throw new IllegalStateException("Unknown endpoint: " + ep + "; cannot map to Accord Node.Id"); + including.add(id); + } + } + else including = null; this.ranges = AccordTopology.toAccordRanges(keyspace, ranges); } @@ -141,6 +160,7 @@ public class AccordRepair private Pair, Long> repairRange(TokenRange range) throws Throwable { List repairedRanges = new ArrayList<>(); + if (shouldAbort != null) throw shouldAbort; @@ -164,7 +184,7 @@ public class AccordRepair long timeoutNanos = getAccordRepairTimeoutNanos(); long maxHlc = AccordService.getBlocking(service.maxConflict(ranges).flatMap(conflict -> { Timestamp conflictMax = mergeMax(conflict, minForEpoch(this.minEpoch.getEpoch())); - return service.sync("[repairId #" + repairId + ']', conflictMax, Ranges.of(range), null, NoLocal, syncRemote, timeoutNanos, NANOSECONDS).map(ignored -> conflictMax.hlc()).chain(); + return service.sync("[repairId #" + repairId + ']', conflictMax, Ranges.of(range), including, NoLocal, syncRemote, timeoutNanos, NANOSECONDS).map(ignored -> conflictMax.hlc()).chain(); }), ranges, bookkeeping, start, start + timeoutNanos); waiting = null; diff --git a/src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java index b3f81bec28..9e67d5dcac 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java @@ -62,14 +62,14 @@ public class RecoverySerializers public void serializeBody(BeginRecovery recover, DataOutputPlus out, Version version) throws IOException { CommandSerializers.partialTxn.serialize(recover.partialTxn, out, version); - int flags = (recover.route != null ? HAS_ROUTE : 0) - | (recover.executeAtOrTxnIdEpoch != recover.txnId.epoch() ? HAS_EXECUTE_AT_EPOCH : 0) - | (recover.isFastPathDecided ? IS_FAST_PATH_DECIDED : 0); + int encodedFlags = (recover.route != null ? HAS_ROUTE : 0) + | (recover.executeAtOrTxnIdEpoch != recover.txnId.epoch() ? HAS_EXECUTE_AT_EPOCH : 0) + | recover.flags << 2; CommandSerializers.ballot.serialize(recover.ballot, out); - out.writeUnsignedVInt32(flags); + out.writeUnsignedVInt32(encodedFlags); if (recover.route != null) KeySerializers.fullRoute.serialize(recover.route, out); - if (0 != (flags & HAS_EXECUTE_AT_EPOCH)) + if (0 != (encodedFlags & HAS_EXECUTE_AT_EPOCH)) out.writeUnsignedVInt(recover.executeAtOrTxnIdEpoch - recover.txnId.epoch()); } @@ -78,15 +78,15 @@ public class RecoverySerializers { PartialTxn partialTxn = CommandSerializers.partialTxn.deserialize(in, version); Ballot ballot = CommandSerializers.ballot.deserialize(in); - int flags = in.readUnsignedVInt32(); + int encodedFlags = in.readUnsignedVInt32(); FullRoute route = null; - if (0 != (flags & HAS_ROUTE)) + if (0 != (encodedFlags & HAS_ROUTE)) route = KeySerializers.fullRoute.deserialize(in); long executeAtOrTxnIdEpoch = txnId.epoch(); - if (0 != (flags & HAS_EXECUTE_AT_EPOCH)) + if (0 != (encodedFlags & HAS_EXECUTE_AT_EPOCH)) executeAtOrTxnIdEpoch += in.readUnsignedVInt32(); - boolean isFastPathDecided = 0 != (flags & IS_FAST_PATH_DECIDED); - return BeginRecovery.SerializationSupport.create(txnId, scope, waitForEpoch, minEpoch, partialTxn, ballot, route, executeAtOrTxnIdEpoch, isFastPathDecided); + int recoveryFlags = encodedFlags >>> 2; + return BeginRecovery.SerializationSupport.create(txnId, scope, waitForEpoch, minEpoch, partialTxn, ballot, route, executeAtOrTxnIdEpoch, recoveryFlags); } @Override diff --git a/src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java index 5d2d5efa46..aa28e4cd27 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java @@ -19,7 +19,6 @@ package org.apache.cassandra.service.accord.serializers; import accord.api.Result; -import accord.primitives.ProgressToken; import org.apache.cassandra.io.UnversionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -27,14 +26,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; public class ResultSerializers { // TODO (desired): this is meant to encode e.g. whether the transaction's condition met or not for clients to later query - public static final Result APPLIED = new Result() - { - @Override - public ProgressToken asProgressToken() - { - return ProgressToken.APPLIED; - } - }; + public static final Result APPLIED = new Result(){}; public static final UnversionedSerializer result = new UnversionedSerializer<>() { diff --git a/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java b/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java index 067c179ac4..0802b083d7 100644 --- a/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java +++ b/src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java @@ -41,8 +41,9 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.accord.TokenRange; import org.apache.cassandra.utils.ArraySerializers; import org.apache.cassandra.utils.CollectionSerializers; -import org.apache.cassandra.utils.LargeBitSetSerializer; +import org.apache.cassandra.utils.SimpleBitSetSerializers; +import static accord.topology.Topology.NO_IDS; import static accord.utils.SortedArrays.fromSimpleBitSet; public class TopologySerializers @@ -112,7 +113,7 @@ public class TopologySerializers range.serialize(shard.range, out); CollectionSerializers.serializeList(shard.nodes, out, nodeId); CollectionSerializers.serializeList(shard.notInFastPath, out, nodeId); - CollectionSerializers.serializeList(shard.joining, out, nodeId); + out.writeUnsignedVInt32(0); // was joining collection, can now be encoding flag bits out.writeUnsignedVInt32(shard.flags().bitset()); } @@ -122,9 +123,9 @@ public class TopologySerializers Range range = ShardSerializer.this.range.deserialize(in); SortedArrayList nodes = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new); SortedArrayList notInFastPath = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new); - SortedArrayList joining = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new); + in.readUnsignedVInt32(); int flags = in.readUnsignedVInt32(); - return Shard.SerializerSupport.create(range, nodes, notInFastPath, joining, new TinyEnumSet<>(flags)); + return Shard.SerializerSupport.create(range, nodes, notInFastPath, NO_IDS, new TinyEnumSet<>(flags)); } @Override @@ -133,12 +134,46 @@ public class TopologySerializers long size = range.serializedSize(shard.range); size += CollectionSerializers.serializedListSize(shard.nodes, nodeId); size += CollectionSerializers.serializedListSize(shard.notInFastPath, nodeId); - size += CollectionSerializers.serializedListSize(shard.joining, nodeId); + size += TypeSizes.sizeofUnsignedVInt(0); size += TypeSizes.sizeofUnsignedVInt(shard.flags().bitset()); return size; } } + private static final int HAS_STALE_IDS = 0x1; + private static final int HAS_HARD_REMOVED_IDS = 0x2; + + private static void serializeRemovedAndStale(Topology topology, DataOutputPlus out) throws IOException + { + CollectionSerializers.serializeList(topology.removedIds(), out, TopologySerializers.nodeId); + int flags = 0; + if (!topology.staleIds().isEmpty()) + flags |= HAS_STALE_IDS; + if (!topology.hardRemovedIds().isEmpty()) + flags |= HAS_HARD_REMOVED_IDS; + out.writeUnsignedVInt32(flags); + if (!topology.staleIds().isEmpty()) + CollectionSerializers.serializeList(topology.staleIds(), out, TopologySerializers.nodeId); + if (!topology.hardRemovedIds().isEmpty()) + CollectionSerializers.serializeList(topology.hardRemovedIds(), out, TopologySerializers.nodeId); + } + + private static long serializedSizeOfRemovedAndStale(Topology topology) + { + long size = CollectionSerializers.serializedListSize(topology.removedIds(), TopologySerializers.nodeId); + int flags = 0; + if (!topology.staleIds().isEmpty()) + flags |= HAS_STALE_IDS; + if (!topology.hardRemovedIds().isEmpty()) + flags |= HAS_HARD_REMOVED_IDS; + size += TypeSizes.sizeofUnsignedVInt(flags); + if (!topology.staleIds().isEmpty()) + size += CollectionSerializers.serializedListSize(topology.staleIds(), TopologySerializers.nodeId); + if (!topology.hardRemovedIds().isEmpty()) + size += CollectionSerializers.serializedListSize(topology.hardRemovedIds(), TopologySerializers.nodeId); + return size; + } + public static final UnversionedSerializer topology = new UnversionedSerializer<>() { @Override @@ -146,8 +181,7 @@ public class TopologySerializers { out.writeLong(topology.epoch()); CollectionSerializers.serializeList(topology.shards(), out, shard); - CollectionSerializers.serializeCollection(topology.removedIds(), out, TopologySerializers.nodeId); - CollectionSerializers.serializeCollection(topology.staleIds(), out, TopologySerializers.nodeId); + serializeRemovedAndStale(topology, out); } @Override @@ -156,8 +190,14 @@ public class TopologySerializers long epoch = in.readLong(); Shard[] shards = ArraySerializers.deserializeArray(in, shard, Shard[]::new); SortedArrayList removedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); - SortedArrayList staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); - return new Topology(epoch, removedIds, staleIds, shards); + SortedArrayList staleIds = NO_IDS, hardRemovedIds = NO_IDS; + int flags = in.readUnsignedVInt32(); + if ((flags & HAS_STALE_IDS) != 0) + staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + if ((flags & HAS_HARD_REMOVED_IDS) != 0) + hardRemovedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + // we don't currently serialize hardRemoved at the shard level, so we must re-apply here + return new Topology(epoch, removedIds, NO_IDS, staleIds, shards).withHardRemoved(hardRemovedIds); } @Override @@ -166,44 +206,42 @@ public class TopologySerializers long size = 0; size += TypeSizes.LONG_SIZE; // epoch size += CollectionSerializers.serializedListSize(topology.shards(), shard); - size += CollectionSerializers.serializedCollectionSize(topology.removedIds(), TopologySerializers.nodeId); - size += CollectionSerializers.serializedCollectionSize(topology.staleIds(), TopologySerializers.nodeId); + size += serializedSizeOfRemovedAndStale(topology); return size; } }; public static final UnversionedSerializer compactTopology = new UnversionedSerializer<>() { + private final LargeBitSet NO_BITS = new LargeBitSet(0); + + private Object2IntHashMap ranges(Topology topology) + { + // need to loop twice; once to collect ranges, and another to save shards + Object2IntHashMap result = new Object2IntHashMap<>(-2); + for (Shard shard : topology.shards()) + { + TokenRange range = (TokenRange) shard.range; + result.putIfAbsent(range.withTable(TableId.UNDEFINED), -1); + } + int count = 0; + for (Map.Entry e : result.entrySet()) + e.setValue(count++); + return result; + } + @Override public void serialize(Topology topology, DataOutputPlus out) throws IOException { out.writeUnsignedVInt(topology.epoch()); - CollectionSerializers.serializeList(topology.removedIds(), out, TopologySerializers.nodeId); - CollectionSerializers.serializeList(topology.staleIds(), out, TopologySerializers.nodeId); - - List shards = topology.shards(); - - // need to loop twice; once to collect ranges, and another to save shards - Object2IntHashMap ranges; - { - Object2IntHashMap rangesBuilder = new Object2IntHashMap<>(-2); - for (Shard shard : shards) - { - TokenRange range = (TokenRange) shard.range; - rangesBuilder.putIfAbsent(range.withTable(TableId.UNDEFINED), -1); - } - int count = 0; - for (Map.Entry e : rangesBuilder.entrySet()) - e.setValue(count++); - - ranges = rangesBuilder; - } + serializeRemovedAndStale(topology, out); + Object2IntHashMap ranges = ranges(topology); CollectionSerializers.serializeCollection(ranges.keySet(), out, TokenRange.noTableSerializer); - out.writeUnsignedVInt32(shards.size()); + out.writeUnsignedVInt32(topology.shards().size()); TableId activeTableId = null; - for (Shard shard : shards) + for (Shard shard : topology.shards()) { TokenRange range = (TokenRange) shard.range; if (activeTableId == null || !activeTableId.equals(range.table())) @@ -221,9 +259,8 @@ public class TopologySerializers CollectionSerializers.serializeList(shard.nodes, out, TopologySerializers.nodeId); LargeBitSet notInFastPath = SortedArrays.toLargeBitSet(shard.nodes, shard.notInFastPath); - LargeBitSetSerializer.instance.serialize(notInFastPath, out); - LargeBitSet joining = SortedArrays.toLargeBitSet(shard.nodes, shard.joining); - LargeBitSetSerializer.instance.serialize(joining, out); + SimpleBitSetSerializers.large.serialize(notInFastPath, out); + SimpleBitSetSerializers.large.serialize(NO_BITS, out); out.writeUnsignedVInt32(shard.flags().bitset()); } } @@ -232,32 +269,16 @@ public class TopologySerializers public long serializedSize(Topology topology) { long size = TypeSizes.sizeofUnsignedVInt(topology.epoch()); - size += CollectionSerializers.serializedListSize(topology.removedIds(), TopologySerializers.nodeId); - size += CollectionSerializers.serializedListSize(topology.staleIds(), TopologySerializers.nodeId); - - List shards = topology.shards(); + size += serializedSizeOfRemovedAndStale(topology); // need to loop twice; once to collect ranges, and another to save shards - Object2IntHashMap ranges; - { - Object2IntHashMap rangesBuilder = new Object2IntHashMap<>(-2); - for (Shard shard : shards) - { - TokenRange range = (TokenRange) shard.range; - rangesBuilder.putIfAbsent(range.withTable(TableId.UNDEFINED), -1); - } - int count = 0; - for (Map.Entry e : rangesBuilder.entrySet()) - e.setValue(count++); - - ranges = rangesBuilder; - } + Object2IntHashMap ranges = ranges(topology); size += CollectionSerializers.serializedCollectionSize(ranges.keySet(), TokenRange.noTableSerializer); - size += TypeSizes.sizeofUnsignedVInt(shards.size()); + size += TypeSizes.sizeofUnsignedVInt(topology.shards().size()); TableId activeTableId = null; - for (Shard shard : shards) + for (Shard shard : topology.shards()) { TokenRange range = (TokenRange) shard.range; size += TypeSizes.sizeof(true); @@ -272,9 +293,8 @@ public class TopologySerializers size += CollectionSerializers.serializedListSize(shard.nodes, TopologySerializers.nodeId); LargeBitSet notInFastPath = SortedArrays.toLargeBitSet(shard.nodes, shard.notInFastPath); - size += LargeBitSetSerializer.instance.serializedSize(notInFastPath); - LargeBitSet joining = SortedArrays.toLargeBitSet(shard.nodes, shard.joining); - size += LargeBitSetSerializer.instance.serializedSize(joining); + size += SimpleBitSetSerializers.large.serializedSize(notInFastPath); + size += SimpleBitSetSerializers.large.serializedSize(NO_BITS); size += TypeSizes.sizeofUnsignedVInt(shard.flags().bitset()); } return size; @@ -285,7 +305,14 @@ public class TopologySerializers { long epoch = in.readUnsignedVInt(); SortedArrays.SortedArrayList removedIds = SortedArrays.SortedArrayList.copySorted(CollectionSerializers.deserializeList(in, TopologySerializers.nodeId), Node.Id[]::new); - SortedArrays.SortedArrayList staleIds = SortedArrays.SortedArrayList.copySorted(CollectionSerializers.deserializeList(in, TopologySerializers.nodeId), Node.Id[]::new); + SortedArrayList staleIds = NO_IDS, hardRemovedIds = NO_IDS; + { + int flags = in.readUnsignedVInt32(); + if ((flags & HAS_STALE_IDS) != 0) + staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + if ((flags & HAS_HARD_REMOVED_IDS) != 0) + hardRemovedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); + } List ranges = CollectionSerializers.deserializeList(in, TokenRange.noTableSerializer); @@ -301,12 +328,12 @@ public class TopologySerializers TokenRange range = ranges.get(rangeIndex).withTable(activeTableId); SortedArrays.SortedArrayList nodes = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new); - LargeBitSet notInFastPath = LargeBitSetSerializer.instance.deserialize(in); - LargeBitSet joining = LargeBitSetSerializer.instance.deserialize(in); + LargeBitSet notInFastPath = SimpleBitSetSerializers.large.deserialize(in); + SimpleBitSetSerializers.large.deserialize(in); int flags = in.readUnsignedVInt32(); - shards[i] = Shard.SerializerSupport.create(range, nodes, fromSimpleBitSet(nodes, notInFastPath, Node.Id[]::new), fromSimpleBitSet(nodes, joining, Node.Id[]::new), new TinyEnumSet<>(flags)); + shards[i] = Shard.SerializerSupport.create(range, nodes, fromSimpleBitSet(nodes, notInFastPath, Node.Id[]::new), NO_IDS, new TinyEnumSet<>(flags)); } - return new Topology(epoch, removedIds, staleIds, shards); + return new Topology(epoch, removedIds, NO_IDS, staleIds, shards).withHardRemoved(hardRemovedIds); } }; } diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java index c1bf62b6a6..ea3d3c615e 100644 --- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java @@ -197,7 +197,8 @@ public class PaxosBallotTracker // Accord uses repair (which updates this low bound) to discover the minimum HLC that was used by Paxos // after Paxos stops. This bound will generally be in the past so it's fine to update it in Accord // all the time - AccordService.instance().ensureMinHlc(lowBound.unixMicros() + 1); + if (AccordService.isSetup()) + AccordService.instance().ensureMinHlc(lowBound.unixMicros() + 1); } public Ballot getHighBound() diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index aa8c238941..6f4fac9f13 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -21,7 +21,6 @@ package org.apache.cassandra.tcm; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -39,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.local.Node; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; @@ -462,8 +462,8 @@ public class ClusterMetadata tokenMap = tokenMap.unassignTokens(nodeId); Node.Id accordId = AccordTopology.tcmIdToAccord(nodeId); - if (accordStaleReplicas.contains(accordId)) - accordStaleReplicas = accordStaleReplicas.without(Collections.singleton(accordId)); + if (accordStaleReplicas.stale().contains(accordId)) + accordStaleReplicas = accordStaleReplicas.withoutStale(SortedArrayList.ofSorted(accordId)); return this; } @@ -534,8 +534,8 @@ public class ClusterMetadata .withNodeState(replacement, NodeState.JOINED); Node.Id accordId = AccordTopology.tcmIdToAccord(replaced); - if (accordStaleReplicas.contains(accordId)) - accordStaleReplicas = accordStaleReplicas.without(Collections.singleton(accordId)); + if (accordStaleReplicas.stale().contains(accordId)) + accordStaleReplicas = accordStaleReplicas.withoutStale(SortedArrayList.ofSorted(accordId)); return this; } @@ -566,15 +566,21 @@ public class ClusterMetadata return this; } - public Transformer markStaleReplicas(Set ids) + public Transformer markStaleReplicas(SortedArrayList markStale) { - accordStaleReplicas = accordStaleReplicas.withNodeIds(ids); + accordStaleReplicas = accordStaleReplicas.withStale(markStale); return this; } - public Transformer unmarkStaleReplicas(Set ids) + public Transformer markHardRemovedReplicas(SortedArrayList markHardRemoved) { - accordStaleReplicas = accordStaleReplicas.without(ids); + accordStaleReplicas = accordStaleReplicas.withHardRemoved(markHardRemoved); + return this; + } + + public Transformer unmarkStaleReplicas(SortedArrayList unmarkStale) + { + accordStaleReplicas = accordStaleReplicas.withoutStale(unmarkStale); return this; } diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index bb660b9845..dab2d8387c 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -447,7 +447,15 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; } case JOINED: if (StorageService.isReplacingSameAddress()) + { + if (DatabaseDescriptor.getAccordTransactionsEnabled()) + { + // TODO (required): we need to support a mode that changes the NodeId when replacing the same address for accord transaction safety + throw new IllegalStateException("Cannot replace same address when accord transactions are enabled."); + } + ReplaceSameAddress.streamData(self, metadata, shouldBootstrap, finishJoiningRing); + } // JOINED appears before BOOTSTRAPPING & BOOT_REPLACE so we can fall // through when we start as REGISTERED/LEFT and complete a full startup diff --git a/src/java/org/apache/cassandra/tcm/Transformation.java b/src/java/org/apache/cassandra/tcm/Transformation.java index 928ce59aca..fc519e6edb 100644 --- a/src/java/org/apache/cassandra/tcm/Transformation.java +++ b/src/java/org/apache/cassandra/tcm/Transformation.java @@ -38,6 +38,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.tcm.transformations.AccordMarkHardRemoved; import org.apache.cassandra.tcm.transformations.AccordMarkRejoining; import org.apache.cassandra.tcm.transformations.AccordMarkStale; import org.apache.cassandra.tcm.transformations.AlterSchema; @@ -250,6 +251,7 @@ public interface Transformation ACCORD_MARK_REJOINING(40, () -> AccordMarkRejoining.serializer), PREPARE_DROP_ACCORD_TABLE(41, () -> PrepareDropAccordTable.serializer), FINISH_DROP_ACCORD_TABLE(42, () -> FinishDropAccordTable.serializer), + ACCORD_MARK_HARD_REMOVED(43, () -> AccordMarkHardRemoved.serializer), ; private final Supplier> serializer; diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index a40a42dded..70664213a2 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -19,7 +19,9 @@ package org.apache.cassandra.tcm.sequences; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.StreamSupport; @@ -29,6 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.api.ConfigurationService.EpochReady; +import accord.local.Node; +import accord.utils.async.AsyncResult; import com.googlecode.concurrenttrees.common.Iterables; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; @@ -44,7 +48,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepairUtils; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordService; -import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; @@ -362,9 +366,29 @@ public class BootstrapAndJoin extends MultiStepOperation } StorageService.instance.repairPaxosForTopologyChange("bootstrap"); - Future bootstrapStream = StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements); - Future accordReady = AccordService.instance().epochReadyFor(metadata, EpochReady::reads); - Future ready = FutureCombiner.allOf(bootstrapStream, accordReady); + List> bootstraps = new ArrayList<>(); + if (AccordService.instance().isEnabled()) + { + IAccordService service = AccordService.instance(); + { + Future ready = AccordService.toFuture(service.epochReadyFor(metadata, EpochReady::metadata)); + logger.info("Waiting for Accord metadata to be ready"); + ready.syncThrowUncheckedOnInterrupt(); + ready.rethrowIfFailed(); + } + + logger.info("Accord metadata is ready, continuing with bootstrap"); + bootstraps.add(service.epochReadyFor(metadata, EpochReady::reads)); + + Node node = service.node(); + node.commandStores().forAllUnsafe(commandStore -> { + AsyncResult ready = commandStore.resumeBootstrap(node); + bootstraps.add(AccordService.toFuture(ready.flatMap(e -> e.reads))); + }); + } + bootstraps.add(StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements)); + + Future ready = FutureCombiner.allOf(bootstraps); try { diff --git a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkHardRemoved.java b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkHardRemoved.java new file mode 100644 index 0000000000..3342678aef --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkHardRemoved.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.transformations; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import accord.local.Node; +import accord.utils.SortedArrays.SortedArrayList; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.service.accord.AccordTopology; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.MultiStepOperation; +import org.apache.cassandra.tcm.Transformation; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.sequences.BootstrapAndReplace; +import org.apache.cassandra.tcm.sequences.LockedRanges; +import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.utils.CollectionSerializers; + +import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; + +public class AccordMarkHardRemoved implements Transformation +{ + private static final Logger logger = LoggerFactory.getLogger(AccordMarkHardRemoved.class); + + private final Set ids; + private final boolean force; + + public AccordMarkHardRemoved(Set ids, boolean force) + { + this.ids = ids; + this.force = force; + } + + @Override + public Kind kind() + { + return Kind.ACCORD_MARK_HARD_REMOVED; + } + + @Override + public Result execute(ClusterMetadata prev) + { + for (NodeId id : ids) + { + if (!prev.directory.peerIds().contains(id)) + continue; + + if (!force) + { + boolean removing = false; + for (MultiStepOperation operation : prev.inProgressSequences) + { + switch (operation.kind()) + { + case REPLACE: + removing |= id.equals(((BootstrapAndReplace) operation).finishReplace.replaced); + continue; + case REMOVE: + case LEAVE: + removing |= id.equals(((UnbootstrapAndLeave) operation).finishLeave.nodeId); + } + } + + if (!removing) + return new Rejected(INVALID, String.format("Cannot mark node %s hard removed as it is still present in the directory.", id)); + } + } + + SortedArrayList hardRemoveIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new)); + + for (Node.Id id : hardRemoveIds) + if (prev.accordStaleReplicas.hardRemoved().contains(id)) + return new Rejected(INVALID, String.format("Cannot mark node %s hard removed as it already is.", id)); + + logger.info("Marking " + ids + " hard removed. These nodes should be permanently offline and unable to respond to any messages at any future point."); + ClusterMetadata.Transformer next = prev.transformer().markHardRemovedReplicas(hardRemoveIds); + return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY); + } + + @Override + public String toString() + { + return "AccordMarkStale{ids=" + ids + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AccordMarkHardRemoved that = (AccordMarkHardRemoved) o; + return Objects.equals(ids, that.ids); + } + + @Override + public int hashCode() + { + return Objects.hash(ids); + } + + public static final AsymmetricMetadataSerializer serializer = new AsymmetricMetadataSerializer<>() + { + @Override + public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException + { + assert t instanceof AccordMarkHardRemoved; + AccordMarkHardRemoved mark = (AccordMarkHardRemoved) t; + CollectionSerializers.serializeCollection(mark.ids, out, version, NodeId.serializer); + out.writeBoolean(mark.force); + } + + @Override + public AccordMarkHardRemoved deserialize(DataInputPlus in, Version version) throws IOException + { + return new AccordMarkHardRemoved(CollectionSerializers.deserializeSet(in, version, NodeId.serializer), in.readBoolean()); + } + + @Override + public long serializedSize(Transformation t, Version version) + { + assert t instanceof AccordMarkHardRemoved; + AccordMarkHardRemoved mark = (AccordMarkHardRemoved) t; + return CollectionSerializers.serializedCollectionSize(mark.ids, version, NodeId.serializer) + TypeSizes.BOOL_SIZE; + } + }; +} diff --git a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkRejoining.java b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkRejoining.java index 402d05cf50..93ebeff232 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkRejoining.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkRejoining.java @@ -21,12 +21,12 @@ package org.apache.cassandra.tcm.transformations; import java.io.IOException; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.local.Node; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.service.accord.AccordTopology; @@ -64,10 +64,10 @@ public class AccordMarkRejoining implements Transformation if (!prev.directory.peerIds().contains(id)) return new Rejected(INVALID, String.format("Can not unmark node %s as it is not present in the directory.", id)); - Set accordIds = ids.stream().map(AccordTopology::tcmIdToAccord).collect(Collectors.toSet()); + SortedArrayList accordIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new)); for (Node.Id id : accordIds) - if (!prev.accordStaleReplicas.contains(id)) + if (!prev.accordStaleReplicas.stale().contains(id)) return new Rejected(INVALID, String.format("Can not unmark node %s as it is not stale.", id)); logger.info("Unmarking " + ids + ". They will now participate in durability status coordination..."); diff --git a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java index 33f68cb65d..aad193be71 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java @@ -19,18 +19,16 @@ package org.apache.cassandra.tcm.transformations; import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import accord.local.Node; import accord.topology.Shard; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -70,10 +68,11 @@ public class AccordMarkStale implements Transformation if (!prev.directory.peerIds().contains(id)) return new Rejected(INVALID, String.format("Can not mark node %s stale as it is not present in the directory.", id)); - Set accordIds = ids.stream().map(AccordTopology::tcmIdToAccord).collect(Collectors.toSet()); + SortedArrayList staleIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new)); + SortedArrayList allStaleIds = prev.accordStaleReplicas.stale().with(staleIds); - for (Node.Id id : accordIds) - if (prev.accordStaleReplicas.contains(id)) + for (Node.Id id : staleIds) + if (prev.accordStaleReplicas.stale().contains(id)) return new Rejected(INVALID, String.format("Can not mark node %s stale as it already is.", id)); for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES)) @@ -82,24 +81,17 @@ public class AccordMarkStale implements Transformation for (AccordTopology.KeyspaceShard shard : shards) { - // We're trying to mark a node in this shard stale... - if (!Collections.disjoint(shard.nodes(), accordIds)) + SortedArrayList intersecting = allStaleIds.intersecting(shard.nodes()); + if (intersecting.size() > Shard.maxToleratedFailures(shard.nodes().size())) { - int quorumSize = Shard.slowQuorumSize(shard.nodes().size()); - Set nonStaleNodes = new HashSet<>(shard.nodes()); - nonStaleNodes.removeAll(accordIds); - nonStaleNodes.removeAll(prev.accordStaleReplicas.ids()); - - // ...but reject the transformation if this would bring us below quorum. - if (nonStaleNodes.size() < quorumSize) - return new Rejected(INVALID, String.format("Can not mark nodes %s stale as that would leave fewer than a quorum of nodes active for ranges %s in keyspace '%s'.", - accordIds, shard.ranges(), keyspace.name)); + return new Rejected(INVALID, String.format("Can not mark nodes %s stale as that would leave fewer than a quorum of nodes active for ranges %s in keyspace '%s'.", + allStaleIds, shard.ranges(), keyspace.name)); } } } logger.info("Marking " + ids + " stale. They will no longer participate in durability status coordination..."); - ClusterMetadata.Transformer next = prev.transformer().markStaleReplicas(accordIds); + ClusterMetadata.Transformer next = prev.transformer().markStaleReplicas(staleIds); return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY); } diff --git a/src/java/org/apache/cassandra/utils/LargeBitSetSerializer.java b/src/java/org/apache/cassandra/utils/LargeBitSetSerializer.java deleted file mode 100644 index 69a13948f7..0000000000 --- a/src/java/org/apache/cassandra/utils/LargeBitSetSerializer.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.utils; - -import java.io.IOException; - -import accord.utils.LargeBitSet; -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.UnversionedSerializer; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputPlus; - -public class LargeBitSetSerializer implements UnversionedSerializer -{ - public static final LargeBitSetSerializer instance = new LargeBitSetSerializer(); - - @Override - public void serialize(LargeBitSet t, DataOutputPlus out) throws IOException - { - long[] raw = LargeBitSet.SerializationSupport.getArray(t); - // find the first word written - int wordsInUse = wordsInUse(raw); - out.writeUnsignedVInt32(raw.length); - out.writeUnsignedVInt32(wordsInUse); - for (int i = 0; i < wordsInUse; i++) - out.writeUnsignedVInt(raw[i]); - } - - @Override - public LargeBitSet deserialize(DataInputPlus in) throws IOException - { - int size = in.readUnsignedVInt32(); - long[] raw = new long[size]; - int wordsInUse = in.readUnsignedVInt32(); - for (int i = 0; i < wordsInUse; i++) - raw[i] = in.readUnsignedVInt(); - return LargeBitSet.SerializationSupport.construct(raw); - } - - @Override - public long serializedSize(LargeBitSet t) - { - long[] raw = LargeBitSet.SerializationSupport.getArray(t); - // find the last word written - int wordsInUse = wordsInUse(raw); - long size = TypeSizes.sizeofUnsignedVInt(raw.length); - size += TypeSizes.sizeofVInt(wordsInUse); - for (int i = 0; i < wordsInUse; i++) - size += TypeSizes.sizeofUnsignedVInt(raw[i]); - return size; - } - - private static int wordsInUse(long[] raw) - { - int wordsInUse = raw.length; - for (int i = raw.length - 1; i >= 0; i--) - { - if (raw[i] != 0) - return wordsInUse; - wordsInUse--; - } - return wordsInUse; - } -} diff --git a/src/java/org/apache/cassandra/utils/MerkleTree.java b/src/java/org/apache/cassandra/utils/MerkleTree.java index e41cd1dff2..9dcdc55901 100644 --- a/src/java/org/apache/cassandra/utils/MerkleTree.java +++ b/src/java/org/apache/cassandra/utils/MerkleTree.java @@ -735,7 +735,7 @@ public class MerkleTree if (offHeapRequested && !offHeapSupported && !warnedOnce) { - logger.warn("Configuration requests off-heap merkle trees, but partitioner does not support it. Ignoring."); + logger.warn("Configuration requests off-heap merkle trees, but partitioner {} does not support it. Ignoring.", partitioner.getClass().getName()); warnedOnce = true; } diff --git a/src/java/org/apache/cassandra/utils/SimpleBitSetSerializers.java b/src/java/org/apache/cassandra/utils/SimpleBitSetSerializers.java new file mode 100644 index 0000000000..10191b8513 --- /dev/null +++ b/src/java/org/apache/cassandra/utils/SimpleBitSetSerializers.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.io.IOException; + +import accord.utils.Invariants; +import accord.utils.LargeBitSet; +import accord.utils.SimpleBitSet; +import accord.utils.SmallBitSet; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.UnversionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public class SimpleBitSetSerializers +{ + public static final AnySerializer any = new AnySerializer(); + public static final SmallSerializer small = new SmallSerializer(); + public static final SmallWithCapacitySerializer smallWithCapacity = new SmallWithCapacitySerializer(); + public static final LargeSerializer large = new LargeSerializer(); + + public static class SmallSerializer implements UnversionedSerializer + { + @Override + public void serialize(SmallBitSet t, DataOutputPlus out) throws IOException + { + out.writeUnsignedVInt(t.bits()); + } + + @Override + public SmallBitSet deserialize(DataInputPlus in) throws IOException + { + return new SmallBitSet(in.readUnsignedVInt()); + } + + @Override + public long serializedSize(SmallBitSet t) + { + return TypeSizes.sizeofUnsignedVInt(t.bits()); + } + } + + public static class SmallWithCapacitySerializer extends SmallSerializer + { + @Override + public void serialize(SmallBitSet t, DataOutputPlus out) throws IOException + { + out.writeUnsignedVInt32(1); + super.serialize(t, out); + } + + @Override + public SmallBitSet deserialize(DataInputPlus in) throws IOException + { + Invariants.require(in.readUnsignedVInt32() <= 1); + return super.deserialize(in); + } + + @Override + public long serializedSize(SmallBitSet t) + { + return TypeSizes.sizeofUnsignedVInt(1) + super.serializedSize(t); + } + } + + public static class LargeSerializer implements UnversionedSerializer + { + @Override + public void serialize(LargeBitSet t, DataOutputPlus out) throws IOException + { + long[] raw = LargeBitSet.SerializationSupport.getArray(t); + out.writeUnsignedVInt32(raw.length); + if (raw.length <= 1) + { + out.writeUnsignedVInt(raw.length == 0 ? 0L : raw[0]); + } + else + { + // find the first word written + int wordsInUse = wordsInUse(raw); + out.writeUnsignedVInt32(wordsInUse); + for (int i = 0; i < wordsInUse; i++) + out.writeUnsignedVInt(raw[i]); + } + } + + @Override + public LargeBitSet deserialize(DataInputPlus in) throws IOException + { + int capacity = in.readUnsignedVInt32(); + return deserializeWithCapacity(capacity, in); + } + + LargeBitSet deserializeWithCapacity(int capacityInLongs, DataInputPlus in) throws IOException + { + if (capacityInLongs <= 1) + { + long v = in.readUnsignedVInt(); + Invariants.require(capacityInLongs == 1 || v == 0); + long[] raw = capacityInLongs == 0 ? new long[0] : new long[] { v }; + return LargeBitSet.SerializationSupport.construct(raw); + } + + long[] raw = new long[capacityInLongs]; + int wordsInUse = in.readUnsignedVInt32(); + for (int i = 0; i < wordsInUse; i++) + raw[i] = in.readUnsignedVInt(); + return LargeBitSet.SerializationSupport.construct(raw); + } + + @Override + public long serializedSize(LargeBitSet t) + { + long[] raw = LargeBitSet.SerializationSupport.getArray(t); + return TypeSizes.sizeofUnsignedVInt(raw.length) + + serializedSizeWithoutCapacity(raw); + } + + private long serializedSizeWithoutCapacity(long[] raw) + { + if (raw.length <= 1) + return TypeSizes.sizeofUnsignedVInt(raw.length == 0 ? 0L : raw[0]); + + // find the last word written + int wordsInUse = wordsInUse(raw); + long size = TypeSizes.sizeofUnsignedVInt(wordsInUse); + for (int i = 0; i < wordsInUse; i++) + size += TypeSizes.sizeofUnsignedVInt(raw[i]); + return size; + } + + private static int wordsInUse(long[] raw) + { + int wordsInUse = raw.length; + for (int i = raw.length - 1; i >= 0; i--) + { + if (raw[i] != 0) + return wordsInUse; + wordsInUse--; + } + return wordsInUse; + } + } + + private static UnversionedSerializer serializer(SimpleBitSet set) + { + if (set.getClass() == LargeBitSet.class) return (UnversionedSerializer) large; + if (set.getClass() == SmallBitSet.class) return (UnversionedSerializer) smallWithCapacity; + throw new UnsupportedOperationException("Unknown bitset type: " + set.getClass()); + } + + public static class AnySerializer implements UnversionedSerializer + { + @Override + public void serialize(SimpleBitSet bitset, DataOutputPlus out) throws IOException + { + serializer(bitset).serialize(bitset, out); + } + + @Override + public SimpleBitSet deserialize(DataInputPlus in) throws IOException + { + int capacity = in.readUnsignedVInt32(); + // can use small or large deserializer for capacity <= 1, but to ensure capacity() is the same we use small serializer only when capacity == 1 + if (capacity == 1) return small.deserialize(in); + else return large.deserializeWithCapacity(capacity, in); + } + + @Override + public long serializedSize(SimpleBitSet bitset) + { + return serializer(bitset).serializedSize(bitset); + } + } + +} diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index a2d0429c3f..3cdc16ce23 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -337,11 +337,26 @@ public class ClusterUtils I toReplace, BiConsumer fn) { - IInstanceConfig toReplaceConf = toReplace.config(); - I inst = addInstance(cluster, toReplaceConf, c -> c.set("auto_bootstrap", true) - .set("progress_barrier_min_consistency_level", ConsistencyLevel.ONE)); - return startHostReplacement(toReplace, inst, fn); + return replaceHostAndStart(cluster, toReplace, fn, ignore -> {}); + } + public static I replaceHostAndStart(AbstractCluster cluster, + I toReplace, + BiConsumer fn, + Consumer configFn) + { + IInstanceConfig toReplaceConf = toReplace.config(); + I inst = addInstance(cluster, toReplaceConf, c -> { + c.set("auto_bootstrap", true) + .set("progress_barrier_min_consistency_level", ConsistencyLevel.ONE); + configFn.accept(c); + }); + return startHostReplacement(toReplace, inst, fn); + } + + public static I startHostReplacement(I toReplace, I inst) + { + return startHostReplacement(toReplace, inst, (i1, i2) -> {}); } /** diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java index 49632ed153..19ba9dd279 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java @@ -24,77 +24,55 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import java.util.function.Function; import org.junit.Assert; import org.junit.Test; -import accord.api.ConfigurationService.EpochReady; import accord.primitives.RoutingKeys; import accord.primitives.Timestamp; import accord.topology.TopologyManager; -import accord.utils.async.AsyncResult; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInvokableInstance; -import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; -import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.AccordCommandStore; -import org.apache.cassandra.service.accord.AccordConfigurationService; import org.apache.cassandra.service.accord.AccordSafeCommandStore; -import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.streaming.StreamEvent; +import org.apache.cassandra.streaming.StreamEventHandler; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamSession; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -import org.assertj.core.api.Assertions; +import org.apache.cassandra.streaming.StreamState; import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.cassandra.Util.spinUntilTrue; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; import static org.apache.cassandra.service.accord.AccordService.getBlocking; -import static org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot.ResultStatus.SUCCESS; -import static org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus.COMPLETED; -public class AccordBootstrapTest extends TestBaseImpl +public class AccordBootstrapTest extends AccordBootstrapTestBase { - private static DecoratedKey dk(int key) - { - IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); - return partitioner.decorateKey(ByteBufferUtil.bytes(key)); - } - - private static PartitionKey pk(int key, String keyspace, String table) - { - TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; - return new PartitionKey(tid, dk(key)); - } - - protected IInvokableInstance bootstrapAndJoinNode(Cluster cluster) + protected IInvokableInstance failedAndResumeBootstrapAndJoinNode(Cluster cluster) { IInstanceConfig config = cluster.newInstanceConfig(); config.set("auto_bootstrap", true); config.set("accord.shard_durability_target_splits", "1"); config.set("accord.shard_durability_cycle", "20s"); + config.set("accord.retry_join_bootstrap", "10s,attempts=1"); + cluster.forEach(instance -> instance.runOnInstance(() -> StreamListener.listener.failStream = true)); IInvokableInstance newInstance = cluster.bootstrap(config); newInstance.startup(cluster); + spinUntilTrue(() -> cluster.stream().anyMatch(instance -> instance.callOnInstance(() -> StreamListener.listener.hasFailedStream))); + newInstance.shutdown(false); + cluster.get(1, 2).forEach(instance -> instance.runOnInstance(() -> StreamListener.listener.failStream = false)); + newInstance.startup(cluster); // todo: re-add once we fix write survey/join ring = false mode // withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000), // () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster))); @@ -103,54 +81,13 @@ public class AccordBootstrapTest extends TestBaseImpl return newInstance; } - private static AccordService service() - { - return (AccordService) AccordService.instance(); - } - - private static void awaitEpoch(long epoch, Function> await) - { - try - { - boolean completed = service().epochReady(Epoch.create(epoch), await).await(60, TimeUnit.SECONDS); - Assertions.assertThat(completed) - .describedAs("Epoch %s did not become ready within timeout on %s -> %s", - epoch, FBUtilities.getBroadcastAddressAndPort(), - service().configService().getEpochSnapshot(epoch)) - .isTrue(); - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } - } - - private static void awaitLocalSyncNotification(long epoch) - { - try - { - AccordConfigurationService configService = service().configService(); - boolean completed = configService.unsafeLocalSyncNotified(epoch).await(30, TimeUnit.SECONDS); - Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s\n%s", - epoch, FBUtilities.getBroadcastAddressAndPort(), service().configService().getDebugStr()), completed); - } - catch (InterruptedException e) - { - throw new RuntimeException(e); - } - } - - private static long maxEpoch(Cluster cluster) - { - return cluster.stream().mapToLong(node -> node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch())).max().getAsLong(); - } - private static class StreamListener implements StreamManager.StreamListener { private static boolean isRegistered = false; private static final StreamListener listener = new StreamListener(); private final List registered = new ArrayList<>(); + private boolean failStream, hasFailedStream; static synchronized void register() { @@ -163,6 +100,34 @@ public class AccordBootstrapTest extends TestBaseImpl public synchronized void onRegister(StreamResultFuture result) { registered.add(result); + if (failStream) + { + result.addEventListener(new StreamEventHandler() + { + @Override + public void handleStreamEvent(StreamEvent event) + { + if (event.eventType == StreamEvent.Type.STREAM_PREPARED) + { + result.getCoordinator().getAllStreamSessions().forEach(StreamSession::abort); + hasFailedStream = true; + } + } + + @Override + public void onSuccess(StreamState result) + { + + } + + @Override + public void onFailure(Throwable t) + { + + } + } + ); + } } public synchronized void forSession(Consumer consumer) @@ -176,13 +141,22 @@ public class AccordBootstrapTest extends TestBaseImpl @Test public void bootstrapTest() throws Throwable { - bootstrapTest(Function.identity(), cluster -> { + bootstrapTest(cluster -> { bootstrapAndJoinNode(cluster); awaitMaxEpochReadyToRead(cluster); }); } - public void bootstrapTest(Function setup, Consumer bootstrapAndJoinNode) throws Throwable + @Test + public void resumeBootstrapTest() throws Throwable + { + bootstrapTest(cluster -> { + failedAndResumeBootstrapAndJoinNode(cluster); + awaitMaxEpochReadyToRead(cluster); + }); + } + + public void bootstrapTest(Consumer bootstrapAndJoinNode) throws Throwable { int originalNodeCount = 2; int expandedNodeCount = originalNodeCount + 1; @@ -286,18 +260,18 @@ public class AccordBootstrapTest extends TestBaseImpl Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); Assert.assertFalse(ss.safeToReadAt().isEmpty()); - Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream() - .filter(entry -> entry.getValue().contains(partitionKey)) - .map(entry -> { - Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); - return entry; - }).count()); - Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream() - .filter(entry -> entry.getValue().contains(partitionKey)) - .map(entry -> { - Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); - return entry; - }).count()); + Assert.assertTrue(ss.bootstrapBeganAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .anyMatch(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return true; + })); + Assert.assertTrue(ss.safeToReadAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .anyMatch(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return true; + })); } })); } @@ -309,131 +283,4 @@ public class AccordBootstrapTest extends TestBaseImpl }); } } - - @Test - public void moveTest() throws Throwable - { - try (Cluster cluster = Cluster.build().withNodes(3) - .withoutVNodes() - .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3)) - .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0")) - .withConfig(config -> config - .set("accord.shard_durability_target_splits", "1") - .set("accord.shard_durability_cycle", "20s") - .with(NETWORK, GOSSIP)) - .start()) - { - cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); - cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); - - long initialMax = maxEpoch(cluster); - long[] tokens = new long[3]; - for (int i=0; i<3; i++) - { - tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens()))); - } - - awaitMaxEpochReadyToRead(cluster); - - for (int key = 0; key < 100; key++) - { - String query = "BEGIN TRANSACTION\n" + - " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + - " SELECT row1.v;\n" + - " IF row1 IS NULL THEN\n" + - " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + - " END IF\n" + - "COMMIT TRANSACTION"; - AccordTestBase.executeWithRetry(cluster, query); - } - - long token = ((tokens[1] - tokens[0]) / 2) + tokens[0]; - long preMove = maxEpoch(cluster); - - cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token))); - - long moveMax = awaitMaxEpochReadyToRead(cluster); - - for (IInvokableInstance node : cluster) - { - node.runOnInstance(() -> { - // validate streaming - List> ranges = StorageService.instance.getLocalRanges("ks"); - TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id; - for (int key = 0; key < 100; key++) - { - DecoratedKey dk = dk(key); - UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key); - if (ranges.stream().anyMatch(range -> range.contains(dk.getToken()))) - { - UntypedResultSet.Row row = getOnlyElement(result); - Assert.assertEquals(key, row.getInt("c")); - Assert.assertEquals(key, row.getInt("v")); - - PartitionKey partitionKey = new PartitionKey(tableId, dk); - - getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> { - if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) - { - AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; - Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); - Assert.assertFalse(ss.safeToReadAt().isEmpty()); - - Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream() - .filter(entry -> entry.getValue().contains(partitionKey)) - .map(entry -> { - Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); - return entry; - }).count()); - Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream() - .filter(entry -> entry.getValue().contains(partitionKey)) - .map(entry -> { - Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); - return entry; - }).count()); - } - })); - } - } - }); - } - } - } - - private static long awaitMaxEpochReadyToRead(Cluster cluster) - { - return awaitMaxEpoch(cluster, EpochReady::reads, true); - } - - private static long awaitMaxEpochMetadataReady(Cluster cluster) - { - return awaitMaxEpoch(cluster, EpochReady::metadata, false); - } - - private static long awaitMaxEpoch(Cluster cluster, SerializableFunction> await, boolean expectReadyToRead) - { - long maxEpoch = maxEpoch(cluster); - for (IInvokableInstance node : cluster) - { - node.acceptOnInstance(aw -> { - ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(maxEpoch)); - Assert.assertEquals(maxEpoch, ClusterMetadata.current().epoch.getEpoch()); - AccordService service = (AccordService) AccordService.instance(); - awaitEpoch(maxEpoch, aw); - AccordConfigurationService configService = service.configService(); - - awaitLocalSyncNotification(maxEpoch); - for (long epoch = configService.minEpoch(); epoch <= maxEpoch; epoch++) - { - Assert.assertEquals(COMPLETED, configService.getEpochSnapshot(maxEpoch).syncStatus); - Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).acknowledged); - Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).received); - if (expectReadyToRead) - Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).reads); - } - }, node.transfer(await)); - } - return maxEpoch; - } - } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTestBase.java new file mode 100644 index 0000000000..57836285e4 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTestBase.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.junit.Assert; + +import accord.api.ConfigurationService.EpochReady; +import accord.utils.async.AsyncResult; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordConfigurationService; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot.ResultStatus.SUCCESS; +import static org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus.COMPLETED; + +public class AccordBootstrapTestBase extends TestBaseImpl +{ + static DecoratedKey dk(int key) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + return partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + static PartitionKey pk(int key, String keyspace, String table) + { + TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; + return new PartitionKey(tid, dk(key)); + } + + protected IInvokableInstance bootstrapAndJoinNode(Cluster cluster) + { + IInstanceConfig config = cluster.newInstanceConfig(); + config.set("auto_bootstrap", true); + config.set("accord.shard_durability_target_splits", "1"); + config.set("accord.shard_durability_cycle", "20s"); + IInvokableInstance newInstance = cluster.bootstrap(config); + newInstance.startup(cluster); + // todo: re-add once we fix write survey/join ring = false mode +// withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000), +// () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster))); +// newInstance.nodetoolResult("join").asserts().success(); + newInstance.nodetoolResult("cms", "describe").asserts().success(); // just make sure we're joined, remove later + return newInstance; + } + + static AccordService service() + { + return (AccordService) AccordService.instance(); + } + + static void awaitEpoch(long epoch, Function> await) + { + try + { + boolean completed = service().epochReady(Epoch.create(epoch), await).await(60, TimeUnit.SECONDS); + Assertions.assertThat(completed) + .describedAs("Epoch %s did not become ready within timeout on %s -> %s", + epoch, FBUtilities.getBroadcastAddressAndPort(), + service().configService().getEpochSnapshot(epoch)) + .isTrue(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + static void awaitLocalSyncNotification(long epoch) + { + try + { + AccordConfigurationService configService = service().configService(); + boolean completed = configService.unsafeLocalSyncNotified(epoch).await(60, TimeUnit.SECONDS); + Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s\n%s", + epoch, FBUtilities.getBroadcastAddressAndPort(), service().configService().getDebugStr()), completed); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + static long maxEpoch(Cluster cluster) + { + return cluster.stream().mapToLong(node -> { + if (node.isShutdown()) + return 0L; + return node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch()); + }).max().getAsLong(); + } + + static long awaitMaxEpochReadyToRead(Cluster cluster) + { + return awaitMaxEpoch(cluster, EpochReady::reads, true); + } + + static long awaitMaxEpochMetadataReady(Cluster cluster) + { + return awaitMaxEpoch(cluster, EpochReady::metadata, false); + } + + static long awaitMaxEpoch(Cluster cluster, SerializableFunction> await, boolean expectReadyToRead) + { + long maxEpoch = maxEpoch(cluster); + for (IInvokableInstance node : cluster) + { + if (node.isShutdown()) + continue; + + node.acceptOnInstance(aw -> { + ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(maxEpoch)); + Assert.assertEquals(maxEpoch, ClusterMetadata.current().epoch.getEpoch()); + AccordService service = (AccordService) AccordService.instance(); + awaitEpoch(maxEpoch, aw); + AccordConfigurationService configService = service.configService(); + + awaitLocalSyncNotification(maxEpoch); + for (long epoch = configService.minEpoch(); epoch <= maxEpoch; epoch++) + { + Assert.assertEquals(COMPLETED, configService.getEpochSnapshot(maxEpoch).syncStatus); + Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).acknowledged); + Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).received); + if (expectReadyToRead) + Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).reads); + } + }, node.transfer(await)); + } + return maxEpoch; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCommandStoreTryExecuteListeningTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCommandStoreTryExecuteListeningTest.java new file mode 100644 index 0000000000..90f22d5abc --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordCommandStoreTryExecuteListeningTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import org.junit.Test; + +import accord.api.Journal; +import accord.local.Command; +import accord.local.Node; +import accord.local.PreLoadContext; +import accord.local.PreLoadContext.Empty; +import accord.local.StoreParticipants; +import accord.primitives.Ballot; +import accord.primitives.Deps; +import accord.primitives.FullRoute; +import accord.primitives.KeyDeps; +import accord.primitives.RangeDeps; +import accord.primitives.Ranges; +import accord.primitives.Routable; +import accord.primitives.RoutingKeys; +import accord.primitives.SaveStatus; +import accord.primitives.Status; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.utils.ImmutableBitSet; +import accord.utils.LargeBitSet; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.serializers.ResultSerializers; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; + +public class AccordCommandStoreTryExecuteListeningTest extends TestBaseImpl +{ + private static DecoratedKey dk(int key) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + return partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + private static PartitionKey pk(int key, String keyspace, String table) + { + TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; + return new PartitionKey(tid, dk(key)); + } + + @Test + public void testTryExecuteListening() throws Throwable + { + try (Cluster cluster = Cluster.build().withNodes(1) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(1, "dc0", "rack0")) + .withConfig(config -> config.set("accord.command_store_shard_count", 1) + .set("accord.queue_shard_count", 1) + .with(NETWORK, GOSSIP)) + .start()) + { + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':1}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); + + cluster.get(1).runOnInstance(() -> { + AccordService service = (AccordService) AccordService.instance(); + + Node node = service.node(); + PartitionKey key = pk(1, "ks", "tbl"); + AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); + + Command txn1a = executed(node, SaveStatus.Applied); + Command txn1b = executed(node, SaveStatus.PreApplied); + Command txn2a = executed(node, SaveStatus.PreApplied, txn1a.txnId()); + Command txn2b = executed(node, SaveStatus.PreApplied, txn1b.txnId()); + Command txn3 = executed(node, SaveStatus.PreApplied, txn1a.txnId(), txn1b.txnId(), txn2b.txnId()); + Command txn4 = executed(node, SaveStatus.PreApplied, txn1a.txnId(), txn1b.txnId(), txn3.txnId()); + Command[] commands = new Command[] { txn1a, txn1b, txn2a, txn2b, txn3, txn4 }; + + AccordService.getBlocking(commandStore.chain((Empty)() -> "Test", safeStore -> { + for (Command command : commands) + commandStore.journal.saveCommand(commandStore.id(), new Journal.CommandUpdate(null, command), () -> {}); + + commandStore.unsafeGetListeners().register(txn1a.txnId(), SaveStatus.Applied, txn2a.txnId()); + commandStore.unsafeGetListeners().register(txn3.txnId(), SaveStatus.Applied, txn4.txnId()); + })); + + AccordService.getBlocking(commandStore.operatorTryToExecuteListeningTxns()); + + for (Command command : commands) + { + Command cmd = AccordService.getBlocking(commandStore.submit(PreLoadContext.contextFor(command.txnId(), "Test"), safeStore -> safeStore.unsafeGet(command.txnId()).current())); + Assertions.assertThat(cmd.saveStatus()).isEqualTo(SaveStatus.Applied); + } + }); + } + } + + private static Command executed(Node node, SaveStatus saveStatus, TxnId ... dependencies) + { + PartitionKey key = pk(1, "ks", "tbl"); + Txn txn = node.agent().emptySystemTxn(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range); + TxnId txnId = node.nextTxnId(txn); + FullRoute route = node.computeRoute(txnId, Ranges.of(key.asRange())); + AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); + int[] rangesToTxnIds = new int[dependencies.length + 1]; + rangesToTxnIds[0] = rangesToTxnIds.length; + for (int i = 1; i < rangesToTxnIds.length ; ++i) + rangesToTxnIds[i] = i - 1; + Deps deps = new Deps(KeyDeps.NONE, RangeDeps.SerializerSupport.create(new accord.primitives.Range[] { key.asRange() }, dependencies, rangesToTxnIds, null)); + Command.WaitingOn waitingOn; { + LargeBitSet waitingOnBits = new LargeBitSet(dependencies.length); + waitingOnBits.setRange(0, dependencies.length); + waitingOn = new Command.WaitingOn(RoutingKeys.EMPTY, deps.rangeDeps, new ImmutableBitSet(waitingOnBits), new ImmutableBitSet(dependencies.length)); + } + return Command.Executed.executed(txnId, saveStatus, Status.Durability.NotDurable, StoreParticipants.execute(commandStore.unsafeGetRangesForEpoch(), route, txnId, txnId.epoch()), Ballot.ZERO, txnId, txn.intersecting(route, true), deps.intersecting(route), Ballot.ZERO, waitingOn, null, ResultSerializers.APPLIED); + } + +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java new file mode 100644 index 0000000000..5669e261a7 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.api.PartitionKey; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.service.accord.AccordService.getBlocking; + +public class AccordMoveTest extends AccordBootstrapTestBase +{ + @Test + public void moveTest() throws Throwable + { + try (Cluster cluster = Cluster.build().withNodes(3) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0")) + .withConfig(config -> config + .set("accord.shard_durability_target_splits", "1") + .set("accord.shard_durability_cycle", "20s") + .with(NETWORK, GOSSIP)) + .start()) + { + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); + + long initialMax = maxEpoch(cluster); + long[] tokens = new long[3]; + for (int i=0; i<3; i++) + { + tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens()))); + } + + awaitMaxEpochReadyToRead(cluster); + + for (int key = 0; key < 100; key++) + { + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + AccordTestBase.executeWithRetry(cluster, query); + } + + long token = ((tokens[1] - tokens[0]) / 2) + tokens[0]; + long preMove = maxEpoch(cluster); + + cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token))); + + long moveMax = awaitMaxEpochReadyToRead(cluster); + + for (IInvokableInstance node : cluster) + { + node.runOnInstance(() -> { + // validate streaming + List> ranges = StorageService.instance.getLocalRanges("ks"); + TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id; + for (int key = 0; key < 100; key++) + { + DecoratedKey dk = dk(key); + UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key); + if (ranges.stream().anyMatch(range -> range.contains(dk.getToken()))) + { + UntypedResultSet.Row row = getOnlyElement(result); + Assert.assertEquals(key, row.getInt("c")); + Assert.assertEquals(key, row.getInt("v")); + + PartitionKey partitionKey = new PartitionKey(tableId, dk); + + getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> { + if (!safeStore.ranges().allAt(preMove).contains(partitionKey)) + { + AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); + Assert.assertFalse(ss.safeToReadAt().isEmpty()); + + Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + } + })); + } + } + }); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordNodetoolTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordNodetoolTest.java index 9af94b783d..5502f7fe5d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordNodetoolTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordNodetoolTest.java @@ -43,7 +43,7 @@ public class AccordNodetoolTest extends TestBaseImpl try (Cluster cluster = init(builder().withNodes(3).withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)).start())) { cluster.get(1).nodetoolResult("accord", "mark_stale", "1").asserts().success(); - cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids())); + cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale())); cluster.get(1).nodetoolResult("accord", "describe").asserts().stdoutContains("Stale Replicas: 1"); // Reject the operation if the target node is already stale: @@ -56,7 +56,7 @@ public class AccordNodetoolTest extends TestBaseImpl cluster.get(1).nodetoolResult("accord", "mark_stale", "4").asserts().failure().errorContains("not present in the directory"); cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1").asserts().success(); - cluster.get(1).runOnInstance(() -> assertEquals(Collections.emptySet(), ClusterMetadata.current().accordStaleReplicas.ids())); + cluster.get(1).runOnInstance(() -> assertEquals(Collections.emptySet(), ClusterMetadata.current().accordStaleReplicas.stale())); cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1").asserts().failure().errorContains("it is not stale"); cluster.get(1).nodetoolResult("accord", "mark_rejoining", "4").asserts().failure().errorContains("not present in the directory"); @@ -72,7 +72,7 @@ public class AccordNodetoolTest extends TestBaseImpl cluster.get(1).nodetoolResult("accord", "mark_stale", "1", "2", "3").asserts().failure().errorContains("that would leave fewer than a quorum"); cluster.get(1).nodetoolResult("accord", "mark_stale", "1", "2").asserts().success(); - cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1), new Node.Id(2)), ClusterMetadata.current().accordStaleReplicas.ids())); + cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1), new Node.Id(2)), ClusterMetadata.current().accordStaleReplicas.stale())); cluster.get(1).nodetoolResult("accord", "describe").asserts().stdoutContains("Stale Replicas: 1,2"); // Reject the operation if a target node is already stale: @@ -89,11 +89,11 @@ public class AccordNodetoolTest extends TestBaseImpl cluster.get(nodeIdToNode.get(2)).shutdown().get(); cluster.get(1).nodetoolResult("removenode", "2", "--force").asserts().success(); cluster.get(1).nodetoolResult("cms", "unregister", "2").asserts().success(); - cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids())); + cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale())); cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1", "3").asserts().failure().errorContains("it is not stale"); cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1", "6").asserts().failure().errorContains("not present in the directory"); - cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids())); + cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale())); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java new file mode 100644 index 0000000000..c6dfc9c382 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord; + +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Test; + +import accord.primitives.RoutingKeys; +import accord.primitives.Timestamp; +import accord.topology.TopologyManager; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.ClusterUtils; +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.accord.AccordOperations; +import org.apache.cassandra.service.accord.AccordSafeCommandStore; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.FBUtilities; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.service.accord.AccordService.getBlocking; + +public class AccordRecoverFromAvailabilityLossTest extends AccordBootstrapTestBase +{ + private static void setConfig(IInstanceConfig config) + { + config.set("accord.command_store_shard_count", 2) + .set("accord.queue_shard_count", 2) + .set("accord.shard_durability_cycle", "20s") + .set("accord.shard_durability_target_splits", "1") + .set("accord.retry_syncpoint", "1s*attempts") + .set("accord.retry_durability", "1s*attempts") + .with(NETWORK, GOSSIP); + } + + @Test + public void replaceWithAvailabilityLossTest() throws Throwable + { + int originalNodeCount = 3; + int expandedNodeCount = originalNodeCount + 1; + + try (Cluster cluster = Cluster.build().withNodes(originalNodeCount) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0")) + .withConfig(AccordRecoverFromAvailabilityLossTest::setConfig) + .start()) + { + cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().reconfigureCMS(ReplicationParams.simpleMeta(3, ClusterMetadata.current().directory.knownDatacenters()))); + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); + + awaitMaxEpochReadyToRead(cluster); + int removeIdx = 3; + int removeId = cluster.get(removeIdx).callOnInstance(() -> ClusterMetadata.current().myNodeId().id()); + + for (int key = 0; key < 50; key++) + { + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + AccordTestBase.executeWithRetry(cluster, query); + } + + FBUtilities.waitOnFuture(cluster.get(removeIdx).shutdown(false)); + + { + List> results = new ArrayList<>(); + for (int key = 50; key < 100; key++) + { + String query = "BEGIN TRANSACTION\n" + + " LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" + + " SELECT row1.v;\n" + + " IF row1 IS NULL THEN\n" + + " INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" + + " END IF\n" + + "COMMIT TRANSACTION"; + + results.add(cluster.coordinator(2).asyncExecuteWithResult(query, ConsistencyLevel.SERIAL)); + } + try { FBUtilities.waitOnFutures(results); } + catch (Throwable t) {} + } + + Future future = cluster.get(2).asyncAcceptsOnInstance((Integer id) -> { + while (true) + { + try + { + AccordOperations.instance.accordMarkHardRemoved(Set.of(new NodeId(id)), false); + break; + } + catch (Throwable t) + { + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + } + }).apply(removeId); + ClusterUtils.replaceHostAndStart(cluster, cluster.get(removeIdx), (i1, i2) -> {}, AccordRecoverFromAvailabilityLossTest::setConfig); + future.get(); + + awaitMaxEpochReadyToRead(cluster); + + cluster.get(4).runOnInstance(() -> { + List> ranges = StorageService.instance.getLocalRanges("ks"); + TopologyManager topologyManager = service().node().topology(); + for (long epoch = topologyManager.minEpoch() ; epoch <= topologyManager.epoch() ; ++epoch) + { + CountDownLatch latch = new CountDownLatch(1); + topologyManager.epochReady(epoch).data.invokeIfSuccess(latch::countDown); + while (true) + { + try + { + if (latch.await(1L, TimeUnit.SECONDS)) + break; + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + } + + for (int key = 0; key < 100; key++) + { + UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key); + PartitionKey partitionKey = pk(key, "ks", "tbl"); + if (ranges.stream().anyMatch(range -> range.contains(partitionKey.token()))) + { + UntypedResultSet.Row row; + if (key < 50) row = getOnlyElement(result); + else try { row = getOnlyElement(result); } catch (NoSuchElementException e) { continue; } + Assert.assertEquals(key, row.getInt("c")); + Assert.assertEquals(key, row.getInt("v")); + + getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> { + if (safeStore.ranges().currentRanges().contains(partitionKey)) + { + AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore; + Assert.assertFalse(ss.bootstrapBeganAt().isEmpty()); + Assert.assertFalse(ss.safeToReadAt().isEmpty()); + + Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream() + .filter(entry -> entry.getValue().contains(partitionKey)) + .map(entry -> { + Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0); + return entry; + }).count()); + } + })); + } + else + { + Assert.assertTrue(result.isEmpty()); + } + } + }); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java new file mode 100644 index 0000000000..84c2bb2a0f --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.accord.journal; + +import org.junit.Test; + +import accord.api.Journal; +import accord.coordinate.CoordinateSyncPoint; +import accord.coordinate.ExecuteSyncPoint; +import accord.local.Command; +import accord.local.Node; +import accord.local.RedundantBefore; +import accord.local.RedundantStatus; +import accord.local.StoreParticipants; +import accord.primitives.Ballot; +import accord.primitives.Deps; +import accord.primitives.FullRoute; +import accord.primitives.KeyDeps; +import accord.primitives.Keys; +import accord.primitives.RangeDeps; +import accord.primitives.Ranges; +import accord.primitives.Routable; +import accord.primitives.RoutingKeys; +import accord.primitives.SaveStatus; +import accord.primitives.Status; +import accord.primitives.SyncPoint; +import accord.primitives.Txn; +import accord.primitives.TxnId; +import accord.primitives.Writes; +import accord.utils.ImmutableBitSet; +import accord.utils.LargeBitSet; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.accord.AccordCommandStore; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.AccordTestUtils; +import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.accord.serializers.ResultSerializers; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.Util.spinUntilTrue; +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; + +public class AccordJournalReplayTest extends TestBaseImpl +{ + private static DecoratedKey dk(int key) + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + return partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + private static PartitionKey pk(int key, String keyspace, String table) + { + TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; + return new PartitionKey(tid, dk(key)); + } + + @Test + public void replayCommandWithOnlyDurableSyncPointDependency() throws Throwable + { + try (Cluster cluster = Cluster.build().withNodes(1) + .withoutVNodes() + .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1)) + .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(1, "dc0", "rack0")) + .withConfig(config -> config.set("accord.command_store_shard_count", 2) + .set("accord.queue_shard_count", 2) + .set("accord.shard_durability_cycle", "20s") + .set("accord.shard_durability_target_splits", "1") + .set("accord.retry_syncpoint", "1s*attempts") + .set("accord.retry_durability", "1s*attempts") + .with(NETWORK, GOSSIP)) + .start()) + { + cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':1}"); + cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'"); + + String txnIdStr = cluster.get(1).callOnInstance(() -> { + AccordService service = (AccordService) AccordService.instance(); + PartitionKey key = pk(1, "ks", "tbl"); + + Node node = service.node(); + Txn txn = AccordTestUtils.createTxn("BEGIN TRANSACTION\n" + + "INSERT INTO ks.tbl (k, c, v) VALUES (?, ?, ?);\n" + + "COMMIT TRANSACTION", 1, 1, 1); + + Txn syncPointTxn = node.agent().emptySystemTxn(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range); + TxnId syncPointId = node.nextTxnId(syncPointTxn); + TxnId txnId = node.nextTxnId(txn); + FullRoute route = node.computeRoute(txnId, txn.keys()); + AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable()); + Deps deps = new Deps(KeyDeps.NONE, RangeDeps.SerializerSupport.create(new accord.primitives.Range[] { key.asRange() }, new TxnId[] { syncPointId }, new int[] { 2, 0 }, new int[] { 2, 0 })); + Command.WaitingOn waitingOn; { + LargeBitSet waitingOnBits = new LargeBitSet(1); + waitingOnBits.set(0); + waitingOn = new Command.WaitingOn(RoutingKeys.EMPTY, deps.rangeDeps, new ImmutableBitSet(waitingOnBits), null); + } + Writes writes = new Writes(txnId, txnId, Keys.of(key), null); + Command command = Command.Executed.executed(txnId, SaveStatus.PreApplied, Status.Durability.NotDurable, StoreParticipants.execute(commandStore.unsafeGetRangesForEpoch(), route, txnId, txnId.epoch()), Ballot.ZERO, txnId, txn.intersecting(route, true), deps.intersecting(route), Ballot.ZERO, waitingOn, writes, ResultSerializers.APPLIED); + commandStore.journal.saveCommand(commandStore.id(), new Journal.CommandUpdate(null, command), () -> {}); + + SyncPoint syncPoint = AccordService.getBlocking(CoordinateSyncPoint.exclusive(node, syncPointId, Ranges.of(key.asRange()))); + AccordService.getBlocking(ExecuteSyncPoint.coordinate(node, syncPoint, 1).onQuorum()); + Keyspace.open("ks").getColumnFamilyStore("tbl").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + Keyspace.open("system_accord").getColumnFamilyStore("commands_for_key").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + spinUntilTrue(() -> { + RedundantBefore.Bounds bounds = commandStore.unsafeGetRedundantBefore().get(key); + return bounds.maxBound(RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE).equals(syncPointId) + && bounds.maxBound(RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE).equals(syncPointId); + }); + return txnId.toString(); + }); + + cluster.get(1).shutdown(false); + cluster.get(1).startup(); + cluster.get(1).runOnInstance(() -> { + AccordService service = (AccordService) AccordService.instance(); + PartitionKey key = pk(1, "ks", "tbl"); + AccordCommandStore commandStore = (AccordCommandStore) service.node().commandStores().unsafeForKey(key.toUnseekable()); + TxnId txnId = TxnId.parse(txnIdStr); + Command command = commandStore.loadCommand(txnId); + Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Applied); + }); + } + } + +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest.java index 88d478fc3c..53a94bba3b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; import net.bytebuddy.implementation.MethodDelegation; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.Constants; import org.apache.cassandra.distributed.api.Feature; @@ -53,6 +54,7 @@ public class NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest extends Te @Test public void test() throws IOException, InterruptedException { + CassandraRelevantProperties.DTEST_ACCORD_ENABLED.setBoolean(false); // when added, accord could not replace same address safely TokenSupplier even = TokenSupplier.evenlyDistributedTokens(2); try (Cluster cluster = init(Cluster.build(2) .withConfig(c -> c.with(Feature.values()).set(Constants.KEY_DTEST_API_STARTUP_FAILURE_AS_SHUTDOWN, false)) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java index 42630b3a29..9246a7df00 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java @@ -91,7 +91,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase if (!prediction.state.get().isWriteTargetFor(token, prediction.node(6).matcher)) continue; - simulatedCluster.waitForQuiescense(); + simulatedCluster.waitForQuiescence(); List replicas = simulatedCluster.state.get().writePlacementsFor(token); // At most 2 replicas should respond, so that when the pending node is added, results would be insufficient for recomputed blockFor BooleanSupplier shouldRespond = atMostResponses(simulatedCluster.state.get().isWriteTargetFor(token, simulatedCluster.node(1).matcher) ? 1 : 2); @@ -121,7 +121,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase .prepareJoin() .startJoin(); - simulatedCluster.waitForQuiescense(); + simulatedCluster.waitForQuiescence(); waiting.forEach(WaitingAction::resume); @@ -165,7 +165,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase !simulatedCluster.state.get().isReadReplicaFor(token(pk), simulatedCluster.node(1).matcher)) continue; - simulatedCluster.waitForQuiescense(); + simulatedCluster.waitForQuiescence(); List replicas = simulatedCluster.state.get().readReplicasFor(token(pk)); Function shouldRespond = respondFrom(1, 4); @@ -185,7 +185,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase .midLeave() .finishLeave(); - simulatedCluster.waitForQuiescense(); + simulatedCluster.waitForQuiescence(); waiting.forEach(WaitingAction::resume); @@ -224,7 +224,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase .prepareJoin() .startJoin(); - simulatedCluster.waitForQuiescense(); + simulatedCluster.waitForQuiescence(); AtomicInteger reads = new AtomicInteger(); AtomicInteger writes = new AtomicInteger(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java index 6613add2e8..3a7059c91f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java @@ -831,7 +831,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase return nodes.values().stream().filter(predicate); } - public void waitForQuiescense() + public void waitForQuiescence() { Epoch waitFor = ClusterMetadataService.instance().log().waitForHighestConsecutive().epoch; realCluster.get(1).acceptsOnInstance((Long epoch) -> { diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java index d75307514d..82a444c37d 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java @@ -383,14 +383,14 @@ public class AccordTopologyMixupTest extends TopologyMixupTestBase= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE); + private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ = + String.format("SELECT * FROM %s.%s WHERE table_id = ? AND quorum_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE); - private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE = - String.format("SELECT * FROM %s.%s WHERE node_id = ? AND quorum_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE); + private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE = + String.format("SELECT * FROM %s.%s WHERE node_id = ? AND table_id = ? AND quorum_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE); - private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ = - String.format("SELECT * FROM %s.%s WHERE shard_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE); + private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ = + String.format("SELECT * FROM %s.%s WHERE table_id = ? AND shard_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE); - private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE = - String.format("SELECT * FROM %s.%s WHERE node_id = ? AND shard_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE); + private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE = + String.format("SELECT * FROM %s.%s WHERE node_id = ? AND table_id = ? AND shard_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE); private static final String SET_PATTERN_TRACE = String.format("UPDATE %s.%s SET bucket_mode = ?, bucket_seen = ?, bucket_size = ?, chance = ?, if_intersects = ?, if_kind = ?, on_failure = ?, on_new = ?, trace_bucket_mode = ?, trace_bucket_size = ?, trace_bucket_sub_size = ?, trace_events = ? WHERE id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.TXN_PATTERN_TRACE); @@ -424,16 +420,40 @@ public class AccordDebugKeyspaceTest extends CQLTester safeStore.commandStore().markShardDurable(safeStore, syncId2, ranges2, HasOutcome.Quorum); })); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE).size()).isGreaterThan(0); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ, syncId1.toString()).size()).isEqualTo(2); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ, syncId2.toString()).size()).isEqualTo(1); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ, syncId1.toString()).size()).isEqualTo(1); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ, syncId2.toString()).size()).isEqualTo(0); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_REMOTE, nodeId).size()).isGreaterThan(0); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE, nodeId, syncId1.toString()).size()).isEqualTo(2); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE, nodeId, syncId2.toString()).size()).isEqualTo(1); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE, nodeId, syncId1.toString()).size()).isEqualTo(1); - Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE, nodeId, syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE, tableId.toString()).size()).isGreaterThan(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " >= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(2); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " >= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " >= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " >= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " > ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " > ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " > ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " > ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " <= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " <= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " <= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " <= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " < ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(2); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " < ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " < ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " < ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_REMOTE, nodeId, tableId.toString()).size()).isGreaterThan(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(2); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(1); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(2); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(3); + Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4); } @Test @@ -453,7 +473,7 @@ public class AccordDebugKeyspaceTest extends CQLTester } @Test - public void completedTxn() throws ExecutionException, InterruptedException + public void completedTxn() { String tableName = createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode = 'full'"); AccordService accord = accord(); @@ -470,9 +490,9 @@ public class AccordDebugKeyspaceTest extends CQLTester getBlocking(accord.node().coordinate(id, txn)); filter.apply.awaitThrowUncheckedOnInterrupt(); spinUntilSuccess(() -> assertRows(execute(QUERY_TXN_BLOCKED_BY, id.toString()), - row(id.toString(), anyInt(), 0, "", "", any(), anyOf(SaveStatus.ReadyToExecute.name(), SaveStatus.Applying.name(), SaveStatus.Applied.name())))); - assertRows(execute(QUERY_TXN, id.toString()), row(id.toString(), anyOf("Applied", "Applying"))); - assertRows(execute(QUERY_TXN_REMOTE, nodeId, id.toString()), row(id.toString(), anyOf("Applied", "Applying"))); + row(id.toString(), anyInt(), 0, "", "", any(), "Applied"))); + assertRows(execute(QUERY_TXN, id.toString()), row(id.toString(), "Applied")); + assertRows(execute(QUERY_TXN_REMOTE, nodeId, id.toString()), row(id.toString(), "Applied")); assertRows(execute(QUERY_JOURNAL, id.toString()), row(id.toString(), "PreAccepted"), row(id.toString(), "Applying"), row(id.toString(), "Applied"), row(id.toString(), null)); assertRows(execute(QUERY_JOURNAL_REMOTE, nodeId, id.toString()), row(id.toString(), "PreAccepted"), row(id.toString(), "Applying"), row(id.toString(), "Applied"), row(id.toString(), null)); assertRows(execute(QUERY_COMMANDS_FOR_KEY, keyStr), row(id.toString(), "APPLIED_DURABLE")); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java index d01817e88e..04fa068f06 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java @@ -61,6 +61,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.service.accord.EndpointMapping.Updateable; import org.apache.cassandra.service.accord.api.AccordAgent; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ValidatingClusterMetadataService; @@ -70,7 +71,6 @@ import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.serialization.Version; -import org.apache.cassandra.utils.MockFailureDetector; import org.apache.cassandra.utils.concurrent.Future; import static accord.impl.AbstractConfigurationServiceTest.TestListener; @@ -170,23 +170,23 @@ public class AccordConfigurationServiceTest try { journal = initJournal(); - AccordConfigurationService service = new AccordConfigurationService(ID1, new AccordAgent(), new Messaging(), new MockFailureDetector(), ScheduledExecutors.scheduledTasks); + AccordConfigurationService service = new AccordConfigurationService(ID1, new AccordAgent(), new Updateable(), new Messaging(), ScheduledExecutors.scheduledTasks); AccordJournal journal_ = journal; TestListener listener = new TestListener(service, true) { @Override - public AsyncResult onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync) + public AsyncResult onTopologyUpdate(Topology topology) { // Fake journal save journal_.saveTopology(new TopologyUpdate(new Int2ObjectHashMap<>(), topology), () -> {}); - return super.onTopologyUpdate(topology, isLoad, startSync); + return super.onTopologyUpdate(topology); } }; service.registerListener(listener); service.start(); Topology topology1 = createTopology(cms); - service.updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1)); + ((Updateable)service.endpointMapper()).updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1)); service.reportTopology(topology1); service.receiveRemoteSyncComplete(ID1, 1); service.receiveRemoteSyncComplete(ID2, 1); @@ -199,8 +199,8 @@ public class AccordConfigurationServiceTest Topology topology3 = createTopology(cms); service.reportTopology(topology3); - AccordConfigurationService loaded = new AccordConfigurationService(ID1, new AccordAgent(), new Messaging(), new MockFailureDetector(), ScheduledExecutors.scheduledTasks); - loaded.updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1)); + AccordConfigurationService loaded = new AccordConfigurationService(ID1, new AccordAgent(), new Updateable(), new Messaging(), ScheduledExecutors.scheduledTasks); + ((EndpointMapping.Updateable)loaded.endpointMapper()).updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1)); listener = new AbstractConfigurationServiceTest.TestListener(loaded, true); loaded.registerListener(listener); journal_.closeCurrentSegmentForTestingIfNonEmpty(); diff --git a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java index 061eab4d6f..0c107e0f7c 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java @@ -87,7 +87,7 @@ public class AccordMessageSinkTest checkRequestReplies(request, new AbstractFetchCoordinator.FetchResponse(null, null, id), - CommitOrReadNack.Insufficient); + CommitOrReadNack.InsufficientAndWaiting); } @@ -98,7 +98,7 @@ public class AccordMessageSinkTest Request request = new ReadTxnData(node, topologies, txnId, topology.ranges(), null, null, txnId.epoch()); checkRequestReplies(request, new ReadData.ReadOk(null, null, 0), - CommitOrReadNack.Insufficient); + CommitOrReadNack.InsufficientAndWaiting); } private static void checkRequestReplies(Request request, Reply... replies) diff --git a/test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java b/test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java index 870c989f47..6f4c6f2402 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java @@ -27,6 +27,7 @@ import accord.local.Node; import accord.utils.AccordGens; import accord.utils.Gen; import accord.utils.Gens; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.tcm.Epoch; @@ -52,8 +53,8 @@ public class AccordStaleReplicasTest qt().check(rs -> { Epoch epoch = epochGen.next(rs); - Set nodes = nodesGen.next(rs); - AsymmetricMetadataSerializers.testSerde(buffer, AccordStaleReplicas.serializer, new AccordStaleReplicas(nodes, epoch), Version.MIN_ACCORD_VERSION); + SortedArrayList nodes = SortedArrayList.copyUnsorted(nodesGen.next(rs), Node.Id[]::new); + AsymmetricMetadataSerializers.testSerde(buffer, AccordStaleReplicas.serializer, new AccordStaleReplicas(nodes, SortedArrayList.ofSorted(), epoch), Version.MIN_ACCORD_VERSION); }); } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java b/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java index 081aaeec47..75b59b3f94 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java @@ -33,9 +33,10 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.annotation.Nullable; + import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.junit.BeforeClass; import org.junit.Test; @@ -56,6 +57,9 @@ import accord.utils.AccordGens; import accord.utils.Gen; import accord.utils.Gens; import accord.utils.RandomSource; +import accord.utils.SortedArrays.SortedArrayList; +import accord.utils.SortedList; +import accord.utils.SortedListSet; import org.apache.cassandra.concurrent.AdaptingScheduledExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; @@ -63,8 +67,6 @@ import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.gms.EndpointState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.HeartBeatState; -import org.apache.cassandra.gms.IFailureDetectionEventListener; -import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.ConnectionType; import org.apache.cassandra.net.Message; @@ -100,8 +102,8 @@ public class AccordSyncPropagatorTest // so when instances are created here they are added to gossip to trick the membership check... Gossiper.instance.clearUnsafe(); - List nodes = nodesGen.next(rs); - Set nodesAsSet = ImmutableSet.copyOf(nodes); + SortedList nodes = SortedArrayList.copyUnsorted(nodesGen.next(rs), Node.Id[]::new); + SortedListSet nodesAsSet = SortedListSet.allOf(nodes); List failures = new ArrayList<>(); RandomDelayQueue delayQueue = new RandomDelayQueue.Factory(rs).get(); @@ -123,13 +125,16 @@ public class AccordSyncPropagatorTest allRanges.put(epoch, ranges); scheduler.schedule(() -> { for (Node.Id nodeId : nodes) - cluster.node(nodeId).propagator.reportSyncComplete(epoch, nodes, nodeId); + { + cluster.node(nodeId).configurationService.receiveRemoteSyncComplete(nodeId, epoch); + cluster.node(nodeId).propagator.reportCoordinationReady(epoch, nodes, nodeId); + } for (int j = 0, attempts = rs.nextInt(1, 4); j < attempts; j++) { for (Range range : ranges) { - Cluster.Instace inst = cluster.node(choose(rs, nodes)); + Cluster.Instance inst = cluster.node(choose(rs, nodesAsSet)); scheduler.schedule(() -> { Ranges subrange = Ranges.of(range); inst.propagator.reportClosed(epoch, nodes, subrange); @@ -159,7 +164,7 @@ public class AccordSyncPropagatorTest if (hasPending(cluster)) throw new AssertionError("Unable to make progress: pending syncs on \n" + cluster.instances.values().stream().filter(i -> i.propagator.hasPending()).map(i -> i.propagator.toString()).collect(Collectors.joining("\n"))); - for (Cluster.Instace inst : cluster.instances.values()) + for (Cluster.Instance inst : cluster.instances.values()) { Cluster.ConfigService cs = inst.configurationService; assertSetsEqual(cs.completedEpochs, allRanges.keySet(), "completedEpochs %s", inst.id); @@ -204,7 +209,7 @@ public class AccordSyncPropagatorTest private static class Cluster implements AccordEndpointMapper { private final ImmutableBiMap nodeToAddress; - private final ImmutableMap instances; + private final ImmutableMap instances; private final RandomSource rs; private final ScheduledExecutorPlus scheduler; @@ -215,15 +220,15 @@ public class AccordSyncPropagatorTest this.rs = rs; this.scheduler = scheduler; ImmutableBiMap.Builder nodeToAddress = ImmutableBiMap.builder(); - ImmutableMap.Builder instances = ImmutableMap.builder(); + ImmutableMap.Builder instances = ImmutableMap.builder(); for (Node.Id id : nodes) { InetAddressAndPort address = addressFromInt(id.id); nodeToAddress.put(id, address); - ConfigService cs = new ConfigService(id); + ConfigService cs = new ConfigService(id, nodes); Sink sink = new Sink(id); - IFailureDetector fd = new FailureDetector(address); - instances.put(id, new Instace(id, address, cs, sink, fd, cs, new AccordSyncPropagator(id, Cluster.this, sink, fd, scheduler, cs))); + FailureWrapper fw = new FailureWrapper(Cluster.this, id); + instances.put(id, new Instance(id, cs, sink, new AccordSyncPropagator(id, fw, sink, scheduler, cs))); Gossiper.instance.endpointStateMap.put(address, new EndpointState(HeartBeatState.empty())); } this.nodeToAddress = nodeToAddress.build(); @@ -244,31 +249,43 @@ public class AccordSyncPropagatorTest } } - public Cluster.Instace node(Node.Id id) + public Instance node(Node.Id id) { - Instace instace = instances.get(id); - if (instace == null) + Instance instance = instances.get(id); + if (instance == null) throw new NullPointerException("Unknown id: " + id); - return instace; + return instance; } - public Cluster.Instace node(InetAddressAndPort address) + public Instance node(InetAddressAndPort address) { - return node(mappedId(address)); + return node(mappedIdOrNull(address)); } @Override - public Node.Id mappedIdOrNull(InetAddressAndPort endpoint) + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object ignore) { return nodeToAddress.inverse().get(endpoint); } @Override - public InetAddressAndPort mappedEndpointOrNull(Node.Id id) + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object ignore) { return nodeToAddress.get(id); } + @Override + public Map removedNodes() + { + return Map.of(); + } + + @Override + public NodeStatus nodeStatus(Node.Id id) + { + throw new UnsupportedOperationException(); + } + private enum Action { DELIVER, TIMEOUT, ERROR @@ -309,7 +326,7 @@ public class AccordSyncPropagatorTest throw new IllegalStateException("Unknown action: " + action); } callbacks.put(message.id(), cb); - scheduler.schedule(() -> AccordService.receive(this, node(to).configurationService, (Message) message.withFrom(mappedEndpoint(from))), 500, TimeUnit.MILLISECONDS); + scheduler.schedule(() -> AccordService.receive(this, node(to).configurationService, (Message) message.withFrom(mappedEndpointOrNull(from))), 500, TimeUnit.MILLISECONDS); scheduler.schedule(() -> { RequestCallback removed = callbacks.remove(message.id()); if (removed != null) @@ -355,72 +372,60 @@ public class AccordSyncPropagatorTest } } - private class FailureDetector implements IFailureDetector + private class FailureWrapper implements AccordEndpointMapper { - private final InetAddressAndPort self; - private final Map> nodeRuns = new HashMap<>(); + private final AccordEndpointMapper wrapped; + private final Node.Id self; + private final Map> nodeRuns = new HashMap<>(); - private FailureDetector(InetAddressAndPort self) + private FailureWrapper(AccordEndpointMapper wrapped, Node.Id self) { + this.wrapped = wrapped; this.self = self; } + @Nullable @Override - public boolean isAlive(InetAddressAndPort ep) + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped) { - if (self.equals(ep)) return true; + return wrapped.mappedIdOrNull(endpoint, logIdentityIfUnmapped); + } - return !nodeRuns.computeIfAbsent(ep, ignore -> Gens.bools().biasedRepeatingRuns(.01, rs.nextInt(3, 15))).next(rs); + @Nullable + @Override + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped) + { + return wrapped.mappedEndpointOrNull(id, logIdentityIfUnmapped); } @Override - public void interpret(InetAddressAndPort ep) + public NodeStatus nodeStatus(Node.Id id) { - throw new UnsupportedOperationException(); + if (self.equals(id)) return NodeStatus.HEALTHY; + + return !nodeRuns.computeIfAbsent(id, ignore -> Gens.bools().biasedRepeatingRuns(.01, rs.nextInt(3, 15))).next(rs) ? NodeStatus.HEALTHY : NodeStatus.UNHEALTHY; } @Override - public void report(InetAddressAndPort ep) + public Map removedNodes() { - throw new UnsupportedOperationException(); - } - - @Override - public void remove(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - - @Override - public void forceConviction(InetAddressAndPort ep) - { - throw new UnsupportedOperationException(); - } - - @Override - public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - throw new UnsupportedOperationException(); - } - - @Override - public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - throw new UnsupportedOperationException(); + return Map.of(); } } private class ConfigService extends AbstractTestConfigurationService implements AccordSyncPropagator.Listener { + private final List nodes; private final Map> syncCompletes = new HashMap<>(); private final Map> endpointAcks = new HashMap<>(); private final NavigableSet completedEpochs = Collections.synchronizedNavigableSet(new TreeSet<>()); private final Map closed = new HashMap<>(); private final Map redundant = new HashMap<>(); - private ConfigService(Node.Id node) + private ConfigService(Node.Id node, List nodes) { super(node, new AccordAgent()); + this.nodes = nodes; } @Override @@ -436,10 +441,9 @@ public class AccordSyncPropagatorTest } @Override - protected void onReadyToCoordinate(Topology topology, boolean startSync) + protected void onReadyToCoordinate(Topology topology) { - Set notify = topology.nodes().stream().filter(i -> !localId.equals(i)).collect(Collectors.toSet()); - instances.get(localId).propagator.reportSyncComplete(topology.epoch(), notify, localId); + instances.get(localId).propagator.reportCoordinationReady(topology.epoch(), topology.nodes(), localId); } @Override @@ -464,16 +468,15 @@ public class AccordSyncPropagatorTest @Override public void onEndpointAck(Node.Id id, long epoch) { - endpointAcks.computeIfAbsent(epoch, ignore -> new HashSet<>()).add(id); + Set acks = endpointAcks.computeIfAbsent(epoch, ignore -> new HashSet<>()); + if (acks.add(id) && acks.containsAll(nodes)) + completedEpochs.add(epoch); } @Override - public void onComplete(long epoch) + public void onEndpointNack(Node.Id id, long epoch) { - completedEpochs.add(epoch); - // TODO why do we see multiple calls? -// if (!completedEpochs.add(epoch)) -// throw new IllegalStateException("Completed epoch " + epoch + " multiple times"); + onEndpointAck(id, epoch); } @Override @@ -491,30 +494,21 @@ public class AccordSyncPropagatorTest } } - public class Instace + public class Instance { private final Node.Id id; - private final InetAddressAndPort address; private final ConfigService configurationService; private final Sink messagingService; - private final IFailureDetector failureDetector; - private final AccordSyncPropagator.Listener listener; private final AccordSyncPropagator propagator; - private Instace(Node.Id id, - InetAddressAndPort address, - ConfigService configurationService, - Sink messagingService, - IFailureDetector failureDetector, - AccordSyncPropagator.Listener listener, - AccordSyncPropagator propagator) + private Instance(Node.Id id, + ConfigService configurationService, + Sink messagingService, + AccordSyncPropagator propagator) { this.id = id; - this.address = address; this.configurationService = configurationService; this.messagingService = messagingService; - this.failureDetector = failureDetector; - this.listener = listener; this.propagator = propagator; } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java index 72f84c9e00..93f732099f 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java @@ -382,6 +382,7 @@ public class AccordTestUtils @Override public TopologyManager topology() { throw new UnsupportedOperationException(); } @Override public long currentStamp() { return stamp; } @Override public void updateStamp() {++stamp;} + @Override public boolean isReplaying() { return false; } }; AccordAgent agent = new AccordAgent(); @@ -397,7 +398,7 @@ public class AccordTestUtils holder.add(1, new CommandStores.RangesForEpoch(1, ranges), ranges); AccordCommandStore result = new AccordCommandStore(0, time, agent, null, cs -> new NoOpProgressLog(), - cs -> new DefaultLocalListeners(new NoOpRemoteListeners(), new NoOpNotifySink()), + cs -> new DefaultLocalListeners(null, new NoOpRemoteListeners(), new NoOpNotifySink()), holder, journal, executor); result.unsafeUpdateRangesForEpoch(); return result; @@ -409,7 +410,7 @@ public class AccordTestUtils TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table); TokenRange range = TokenRange.fullRange(metadata.id, metadata.partitioner); Node.Id node = new Id(1); - Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node), Collections.emptySet())); + Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node))); AccordCommandStore store = createAccordCommandStore(node, now, topology); store.execute((PreLoadContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20)); return store; diff --git a/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java b/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java index 7054b35066..94a2556601 100644 --- a/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java @@ -35,7 +35,7 @@ public class EndpointMappingTest { qt().forAll(CassandraGenerators.INET_ADDRESS_AND_PORT_GEN, SourceDSL.integers().between(1, Integer.MAX_VALUE).map(Node.Id::new)).checkAssert((endpoint, id) -> { EndpointMapping mapping = EndpointMapping.builder(1).add(endpoint, id).build(); - Assertions.assertThat(mapping.mappedEndpoint(id)).isEqualTo(endpoint); + Assertions.assertThat(mapping.mappedEndpointOrNull(id)).isEqualTo(endpoint); }); } } diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java index 636f7244d9..406d65aad8 100644 --- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java @@ -42,6 +42,8 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.LongStream; +import javax.annotation.Nullable; + import com.google.common.collect.Sets; import org.junit.Test; import org.slf4j.Logger; @@ -71,8 +73,6 @@ import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; -import org.apache.cassandra.gms.IFailureDetectionEventListener; -import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; @@ -128,7 +128,7 @@ public class EpochSyncTest @Test public void test() { - stateful().withExamples(50).withSteps(500).check(commands(() -> Cluster::new) + stateful().withSeed(1).withExamples(50).withSteps(500).check(commands(() -> Cluster::new) .destroyState(cluster -> { finishPendingWork(cluster); cluster.processAll(); @@ -210,50 +210,32 @@ public class EpochSyncTest private int nodeCounter = 0; private final ValidatingClusterMetadataService cms = ValidatingClusterMetadataService.createAndRegister(NodeVersion.CURRENT_METADATA_VERSION); - private final IFailureDetector fd = new IFailureDetector() + class Mapper implements AccordEndpointMapper { @Override - public boolean isAlive(InetAddressAndPort ep) + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped) { - return instances.get(nodeId(ep)).status != Status.Removed; + return nodeId(endpoint); } @Override - public void interpret(InetAddressAndPort ep) + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped) { - + return address(id); } @Override - public void report(InetAddressAndPort ep) + public Map removedNodes() { - + return Map.of(); } @Override - public void remove(InetAddressAndPort ep) + public NodeStatus nodeStatus(Node.Id id) { - + return instances.get(id).status != Status.Removed ? NodeStatus.HEALTHY : NodeStatus.UNKNOWN; } - - @Override - public void forceConviction(InetAddressAndPort ep) - { - - } - - @Override - public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - - } - - @Override - public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener) - { - - } - }; + } public Cluster(RandomSource rs) { @@ -579,7 +561,7 @@ public class EpochSyncTest ClusterMetadata.Transformer builder = cms.metadata().transformer(); - Instance instance = new Instance(id, token, builder.epoch(), createMessaging(id), fd); + Instance instance = new Instance(id, token, builder.epoch(), createMessaging(id), new Mapper()); instances.put(id, instance); tokens.add(token); @@ -675,7 +657,7 @@ public class EpochSyncTest private final Epoch epoch; private Status status = Status.Init; - Instance(Node.Id node, long token, Epoch epoch, SimulatedMessageDelivery messagingService, IFailureDetector failureDetector) + Instance(Node.Id node, long token, Epoch epoch, SimulatedMessageDelivery messagingService, AccordEndpointMapper mapper) { this.id = node; this.token = token; @@ -683,11 +665,11 @@ public class EpochSyncTest // TODO (review): Should there be a real scheduler here? Is it possible to adapt the Scheduler interface to scheduler used in this test? TimeService time = TimeService.ofNonMonotonic(globalExecutor::currentTimeMillis, TimeUnit.MILLISECONDS); this.topology = new TopologyManager(SizeOfIntersectionSorter.SUPPLIER, new TestAgent.RethrowAgent(), id, time, new DefaultTimeouts(time)); - config = new AccordConfigurationService(node, new AccordAgent(), messagingService, failureDetector, scheduler); + config = new AccordConfigurationService(node, new AccordAgent(), mapper, messagingService, scheduler); config.registerListener(new ConfigurationService.Listener() { @Override - public AsyncResult onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync) + public AsyncResult onTopologyUpdate(Topology topology) { AsyncResult metadata = schedule(rs.nextInt(1, 10), TimeUnit.SECONDS, (Callable) () -> null).beginAsResult(); AsyncResult coordination = metadata.flatMap(ignore -> schedule(rs.nextInt(1, 10), TimeUnit.SECONDS, (Callable) () -> null).beginAsResult()); @@ -699,7 +681,7 @@ public class EpochSyncTest ready.coordinate.invokeIfSuccess(() -> topology().onEpochSyncComplete(id, topology.epoch())); if (topology().minEpoch() == topology.epoch() && topology().epoch() != topology.epoch()) return ready.coordinate; - config.acknowledgeEpoch(ready, startSync); + config.acknowledgeEpoch(ready); return ready.coordinate; } diff --git a/test/unit/org/apache/cassandra/service/accord/SimpleAccordEndpointMapper.java b/test/unit/org/apache/cassandra/service/accord/SimpleAccordEndpointMapper.java index acca53d5fc..9fff97ad03 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimpleAccordEndpointMapper.java +++ b/test/unit/org/apache/cassandra/service/accord/SimpleAccordEndpointMapper.java @@ -21,6 +21,7 @@ package org.apache.cassandra.service.accord; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.util.Map; import accord.local.Node; import org.apache.cassandra.locator.InetAddressAndPort; @@ -31,7 +32,7 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper INSTANCE; @Override - public Node.Id mappedIdOrNull(InetAddressAndPort endpoint) + public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object ignore) { if (endpoint.addressBytes.length != 4) throw new IllegalArgumentException("Only IPV4 is allowed: given " + endpoint.toString(true)); @@ -39,7 +40,7 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper } @Override - public InetAddressAndPort mappedEndpointOrNull(Node.Id id) + public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object ignore) { byte[] array = ByteBufferUtil.bytes(id.id).array(); try @@ -51,4 +52,16 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper throw new AssertionError("Unable to convert " + id + " to an IPV4 address", e); } } + + @Override + public Map removedNodes() + { + return Map.of(); + } + + @Override + public NodeStatus nodeStatus(Node.Id id) + { + return NodeStatus.HEALTHY; + } } diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java index d73763ba5a..fceec176bc 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java @@ -241,6 +241,12 @@ public class SimulatedAccordCommandStore implements AutoCloseable { ++stamp; } + + @Override + public boolean isReplaying() + { + return false; + } }; TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent() @@ -265,7 +271,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable agent, null, ignore -> new ProgressLog.NoOpProgressLog(), - cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink() + cs -> new DefaultLocalListeners(null, new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink() { @Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {} @Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; } @@ -458,7 +464,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable { TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain()); Ballot ballot = Ballot.fromValues(storeService.epoch(), storeService.now(), nodeId); - BeginRecovery br = new BeginRecovery(nodeId, topologies, txnId, null, false, txn, route, ballot); + BeginRecovery br = new BeginRecovery(nodeId, topologies, txnId, null, 0, txn, route, ballot); return Pair.create(txnId, processAsync(br, safe -> { var reply = br.apply(safe); diff --git a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java index 6bf0e569da..272f1e022e 100644 --- a/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java +++ b/test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java @@ -288,7 +288,7 @@ public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester }); var delay = preAcceptAsync.flatMap(ignore -> AsyncChains.chain(instance.unorderedScheduled, () -> { Ballot ballot = Ballot.fromValues(instance.storeService.epoch(), instance.storeService.now(), nodeId); - return new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, null, false, txn, route, ballot); + return new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, null, 0, txn, route, ballot); }).beginAsResult()); var recoverAsync = delay.flatMap(br -> instance.processAsync(br, safe -> { var reply = br.apply(safe); diff --git a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java index edca7e1c6b..01c52656c5 100644 --- a/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java +++ b/test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java @@ -645,7 +645,7 @@ public class CommandsForKeySerializerTest null, null, ignore -> new ProgressLog.NoOpProgressLog(), - ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE), + ignore -> new DefaultLocalListeners(null, new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE), new EpochUpdateHolder()); } @@ -711,6 +711,7 @@ public class CommandsForKeySerializerTest @Override public TopologyManager topology() { return null; } @Override public long currentStamp() { return 0; } @Override public void updateStamp() { throw new UnsupportedOperationException(); } + @Override public boolean isReplaying() { return false; } @Override public long now() { return 0; } @Override public long elapsed(TimeUnit unit) { return 0; } }; } diff --git a/test/unit/org/apache/cassandra/tcm/sequences/DropAccordTableTest.java b/test/unit/org/apache/cassandra/tcm/sequences/DropAccordTableTest.java index 8b18368809..cba63dc1c1 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/DropAccordTableTest.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/DropAccordTableTest.java @@ -37,6 +37,8 @@ import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Types; +import org.apache.cassandra.service.accord.AccordService; +import org.apache.cassandra.service.accord.IAccordService; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -62,7 +64,9 @@ public class DropAccordTableTest { static { - DatabaseDescriptor.clientInitialization(); + DatabaseDescriptor.daemonInitialization(); + // TODO (required): should this be a no-op service? Probably not! But this is what it was before (only relying on !accord.enabled check in AccordService.instance()...) + AccordService.unsafeSetNewAccordService(new IAccordService.NoOpAccordService()); } private static final TransactionalMode[] ACCORD_ENABLED_MODES = Stream.of(TransactionalMode.values()) diff --git a/test/unit/org/apache/cassandra/utils/AccordGenerators.java b/test/unit/org/apache/cassandra/utils/AccordGenerators.java index 2c8f292b68..0fdd6f8dd8 100644 --- a/test/unit/org/apache/cassandra/utils/AccordGenerators.java +++ b/test/unit/org/apache/cassandra/utils/AccordGenerators.java @@ -740,13 +740,11 @@ public class AccordGenerators Gen> shardFlagsGen = shardFlagsGen(); return rs -> { SortedArrayList nodes = nodesGen.next(rs); - int maxFailures = Shard.maxToleratedFailures(nodes.size()); int slowQuorumSize = Shard.slowQuorumSize(nodes.size()); Set fastPathElectorate = new TreeSet<>(select(nodes, nodes.size() == slowQuorumSize ? slowQuorumSize : rs.nextInt(slowQuorumSize, nodes.size())).next(rs)); List nonFastPath = new ArrayList<>(Sets.difference(new HashSet<>(nodes), fastPathElectorate)); nonFastPath.sort(Comparator.naturalOrder()); - Set joining = new TreeSet<>(select(nonFastPath, nonFastPath.size() == 0 ? 0 : rs.nextInt(0, nonFastPath.size())).next(rs)); - return Shard.create(range, nodes, fastPathElectorate, joining, shardFlagsGen.next(rs)); + return Shard.create(range, nodes, fastPathElectorate, shardFlagsGen.next(rs)); }; } @@ -770,13 +768,19 @@ public class AccordGenerators return rs -> { long epoch = epochGen.nextLong(rs); Ranges ranges = rangesGen.next(rs); - if (ranges.isEmpty()) return new Topology(epoch, new Shard[0]); + if (ranges.isEmpty()) + return new Topology(epoch, new Shard[0]); List shards = new ArrayList<>(ranges.size()); for (Range range : ranges) shards.add(shardGen(range).next(rs)); + //TODO (coverage): staleNodes - return new Topology(epoch, shards.toArray(Shard[]::new)); + Topology topology = new Topology(epoch, shards.toArray(Shard[]::new)); + SortedArrayList nodes = topology.nodes(); + int hardRemovedCount = Math.min(rs.nextBoolean() ? 0 : rs.nextInt(0, 3), nodes.size()); + SortedArrayList hardRemoved = SortedArrayList.copyUnsorted(select(nodes, hardRemovedCount).next(rs), Node.Id[]::new); + return topology.withHardRemoved(hardRemoved); }; } diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index efbc3cabb6..b428e9acb9 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -51,6 +51,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import accord.utils.SortedArrays.SortedArrayList; import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.schema.*; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; @@ -1872,7 +1873,7 @@ public final class CassandraGenerators { Gen> staleIdsGen = Generators.set(accordNodeId(), SourceDSL.integers().between(0, 10)); Gen epochGen = epochs(); - return rnd -> new AccordStaleReplicas(staleIdsGen.generate(rnd), epochGen.generate(rnd)); + return rnd -> new AccordStaleReplicas(SortedArrayList.copyUnsorted(staleIdsGen.generate(rnd), Node.Id[]::new), SortedArrayList.ofSorted(), epochGen.generate(rnd)); } public static Gen accordFastPath() diff --git a/test/unit/org/apache/cassandra/utils/LargeBitSetSerializerTest.java b/test/unit/org/apache/cassandra/utils/LargeBitSetSerializerTest.java index 7019835229..c32b69a5c2 100644 --- a/test/unit/org/apache/cassandra/utils/LargeBitSetSerializerTest.java +++ b/test/unit/org/apache/cassandra/utils/LargeBitSetSerializerTest.java @@ -32,7 +32,7 @@ public class LargeBitSetSerializerTest public void test() { @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) DataOutputBuffer output = new DataOutputBuffer(); - qt().forAll(largeBitSetGen()).check(bits -> Serializers.testSerde(output, LargeBitSetSerializer.instance, bits)); + qt().forAll(largeBitSetGen()).check(bits -> Serializers.testSerde(output, SimpleBitSetSerializers.large, bits)); } private static Gen largeBitSetGen()