natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce "hardRemoved" concept for marking nodes permanently offline, to permit recovery from availability loss 

Also fix:
 - Paxos should not update minHlc if AccordService is not setup
 - Remove EndpointMapper methods that require non-null, to ensure call-sites handle null case explicitly
 - AccordRepair should specify the Node Ids that must participate, but these should not affect the durability requirements for GC
 - Avoid erroneously marking UniversalOrInvalidated when only known to quorum
 - AccordSyncPropagator should not consult FailureDetector as it may throw UnknownEndpointException
 - Refuse ReplaceSameAddress if accord enabled
 - Reset Accord topologies on restart if no local command stores to ensure we can catch up (as intervening epochs may have otherwise been GC'd)
 - MaybeRecover should not abort if found durable if either: 1) prevProgressToken already witnessed durable; or 2) Home state has requested we recover anyway
 - Invariant in MaybeRecover is too strong now we permit recovering when known to be stable
 - ReadData must respond with InsufficientAndWaiting if not Stable to ensure Durability is inferred correctly by ExecuteTxn
 - Avoid StackOverflowException in NotifyWaitingOn
 - PreAccept should honour the REJECTED flag of any member of the latest topology if the vote would be relied upon for any prior epoch's quorum
 - Resume Accord bootstrap on restart
 - Fix burn test non-determinism
 - Reset Accord topologies on restart if no local command stores to ensure we can catch up (as intervening epochs may have otherwise been GC'd)
 - Exclude stale or removed ids from sync points
 - Terminate a failed DurabilityRequest if a required participant is now removed/hardRemoved
Also improve:
 - Introduce accord_debug_keyspace.{listeners_txn, listeners_local}
 - Filter and record faulty in AbstractCoordination
 - If we are the home shard, and our home progress status is done, we should not wait for the home shard to advance the waiting progress state
 - Introduce CommandStore.tryExecuteListening to try to execute all waiting transactions and their dependencies
 - ExecuteTxn on recovery should not wait, to avoid consuming progress log concurrency slots for transactions that cannot execute
 - HomeState should update its phase based on the command status to handle operator request to retry all states
 - Separate home/wait queue in DefaultProgressLog
 - Additional tracing
 - Do not call trySendMore from prerecordFailure
 - Force full recovery for sync points if we have lost quorum

patch by Benedict; reviewed by Alex Petrov for CASSANDRA-20921
This commit is contained in:
Benedict Elliott Smith 2025-09-19 15:17:39 +01:00
parent 0db9d4d926
commit c9e31f297c
79 changed files with 2807 additions and 1087 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit e896c9c328d3e8a3b7ddd85381edeaf1b4b8ccb2
Subproject commit a575ab316b4e79f99009801312579bcbf86451ef

11
src/java/org/apache/cassandra/config/AccordSpec.java
View File

@ -143,11 +143,12 @@ public class AccordSpec
public String slow_syncpoint_preaccept = "10s";
public String slow_txn_preaccept = "30ms <= p50*2 <= 100ms";
public String slow_read = "30ms <= p50*2 <= 100ms";
public StringRetryStrategy retry_syncpoint = new StringRetryStrategy("10s*attempts <= 600s");
public StringRetryStrategy retry_durability = new StringRetryStrategy("10s*attempts <= 600s");
public StringRetryStrategy retry_bootstrap = new StringRetryStrategy("10s*attempts <= 600s");
public StringRetryStrategy retry_fetch_min_epoch = new StringRetryStrategy("200ms...1s*attempts <= 1s,retries=3");
public StringRetryStrategy retry_fetch_topology = new StringRetryStrategy("200ms...1s*attempts <= 1s,retries=100");
public StringRetryStrategy retry_syncpoint = new StringRetryStrategy("10s*attempt <= 600s");
public StringRetryStrategy retry_durability = new StringRetryStrategy("10s*attempt <= 600s");
public StringRetryStrategy retry_bootstrap = new StringRetryStrategy("10s*attempt <= 600s");
public StringRetryStrategy retry_join_bootstrap = new StringRetryStrategy("30s*attempt,attempts=5");
public StringRetryStrategy retry_fetch_min_epoch = new StringRetryStrategy("200ms...1s*attempt <= 1s,retries=3");
public StringRetryStrategy retry_fetch_topology = new StringRetryStrategy("200ms...1s*attempt <= 1s,retries=100");
public StringRetryStrategy retry_journal_index_ready = new StringRetryStrategy("100ms");
public volatile DurationSpec.IntSecondsBound fast_path_update_delay = null;

521
src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java
View File

@ -18,10 +18,13 @@
package org.apache.cassandra.db.virtual;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@ -40,16 +43,21 @@ import javax.annotation.Nullable;
import com.google.common.collect.ImmutableSet;
import accord.api.LocalListeners;
import accord.coordinate.AbstractCoordination;
import accord.coordinate.Coordination;
import accord.coordinate.Coordination.CoordinationKind;
import accord.coordinate.Coordinations;
import accord.coordinate.PrepareRecovery;
import accord.coordinate.tracking.AbstractTracker;
import accord.impl.progresslog.DefaultProgressLog.ModeFlag;
import accord.local.Commands.NotifyWaitingOnPlus;
import accord.local.cfk.CommandsForKey.TxnInfo;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.RoutingKeys;
import accord.topology.Shard;
import accord.topology.Topology;
import accord.utils.SortedListMap;
import accord.utils.TinyEnumSet;
import org.apache.cassandra.db.PartitionPosition;
@ -118,6 +126,7 @@ import org.apache.cassandra.service.accord.AccordCommandStores;
import org.apache.cassandra.service.accord.AccordExecutor;
import org.apache.cassandra.service.accord.AccordJournal;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordOperations;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTracing;
import org.apache.cassandra.service.accord.AccordTracing.BucketMode;
@ -133,6 +142,7 @@ import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.service.consensus.migration.TableMigrationState;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.LocalizeString;
import static accord.coordinate.Infer.InvalidIf.NotKnownToBeInvalid;
@ -159,10 +169,12 @@ import static org.apache.cassandra.db.virtual.VirtualTable.Sorted.UNSORTED;
import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_ACCORD_DEBUG;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
// TODO (expected): split into separate classes in own package
public class AccordDebugKeyspace extends VirtualKeyspace
{
public static final String COMMANDS_FOR_KEY = "commands_for_key";
public static final String COMMANDS_FOR_KEY_UNMANAGED = "commands_for_key_unmanaged";
public static final String COMMAND_STORE_OPS = "command_store_ops";
public static final String CONSTANTS = "constants";
public static final String COORDINATIONS = "coordinations";
public static final String DURABILITY_SERVICE = "durability_service";
@ -170,8 +182,11 @@ public class AccordDebugKeyspace extends VirtualKeyspace
public static final String EXECUTOR_CACHE = "executor_cache";
public static final String EXECUTORS = "executors";
public static final String JOURNAL = "journal";
public static final String LISTENERS_DEPS = "listeners_deps";
public static final String LISTENERS_LOCAL = "listeners_local";
public static final String MAX_CONFLICTS = "max_conflicts";
public static final String MIGRATION_STATE = "migration_state";
public static final String NODE_OPS = "node_ops";
public static final String PROGRESS_LOG = "progress_log";
public static final String REDUNDANT_BEFORE = "redundant_before";
public static final String REJECT_BEFORE = "reject_before";
@ -182,6 +197,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
public static final String TXN_TRACE = "txn_trace";
public static final String TXN_TRACES = "txn_traces";
public static final String TXN_OPS = "txn_ops";
public static final String SHARD_EPOCHS = "shard_epochs";
private static final Function<Object, String> TO_STRING = AccordDebugKeyspace::toStringOrNull;
@ -190,16 +206,21 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private AccordDebugKeyspace()
{
super(VIRTUAL_ACCORD_DEBUG, List.of(
new ExecutorsTable(),
new CoordinationsTable(),
new CommandsForKeyTable(),
new CommandsForKeyUnmanagedTable(),
new CommandStoreOpsTable(),
new ConstantsTable(),
new DurabilityServiceTable(),
new DurableBeforeTable(),
new ExecutorCacheTable(),
new ExecutorsTable(),
new ListenersDepsTable(),
new ListenersLocalTable(),
new JournalTable(),
new MaxConflictsTable(),
new MigrationStateTable(),
new NodeOpsTable(),
new ProgressLogTable(),
new RedundantBeforeTable(),
new RejectBeforeTable(),
@ -210,7 +231,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
new TxnPatternTraceTable(),
new TxnPatternTracesTable(),
new TxnOpsTable(),
new ConstantsTable()
new ShardEpochsTable()
));
}
@ -232,8 +253,14 @@ public class AccordDebugKeyspace extends VirtualKeyspace
for (CoordinationKind coordinationKind : CoordinationKind.values())
dataSet.row("CoordinationKind", coordinationKind.name());
for (TxnOpsTable.Op op : TxnOpsTable.Op.values())
dataSet.row("Op", op.name()).column("description", op.description);
for (TxnOpsTable.TxnOp op : TxnOpsTable.TxnOp.values())
dataSet.row("TxnOp", op.name()).column("description", op.description);
for (NodeOpsTable.NodeOp op : NodeOpsTable.NodeOp.values())
dataSet.row("NodeOp", op.name()).column("description", op.description);
for (CommandStoreOpsTable.CommandStoreOp op : CommandStoreOpsTable.CommandStoreOp.values())
dataSet.row("CommandStoreOp", op.name()).column("description", op.description);
}
@Override
@ -268,7 +295,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
AccordCommandStores commandStores = (AccordCommandStores) AccordService.instance().node().commandStores();
AccordCommandStores commandStores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores();
// TODO (desired): we can easily also support sorted collection for DESC queries
for (AccordExecutor executor : commandStores.executors())
{
@ -307,8 +334,8 @@ public class AccordDebugKeyspace extends VirtualKeyspace
"Accord Coordination State",
"CREATE TABLE %s (\n" +
" txn_id 'TxnIdUtf8Type',\n" +
" kind text,\n" +
" coordination_id bigint,\n" +
" kind text,\n" +
" description text,\n" +
" nodes text,\n" +
" nodes_inflight text,\n" +
@ -316,29 +343,44 @@ public class AccordDebugKeyspace extends VirtualKeyspace
" participants text,\n" +
" replies text,\n" +
" tracker text,\n" +
" PRIMARY KEY (txn_id, kind, coordination_id)" +
')', TxnIdUtf8Type.instance), FAIL, UNSORTED);
" PRIMARY KEY (txn_id, coordination_id, kind)" +
')', TxnIdUtf8Type.instance), FAIL, SORTED);
}
@Override
public void collect(PartitionsCollector collector)
{
Coordinations coordinations = AccordService.instance().node().coordinations();
List<Coordination> coordinations = new ArrayList<>();
for (Coordination c : AccordService.unsafeInstance().node().coordinations())
coordinations.add(c);
Comparator<Coordination> comparator = CoordinationsTable::compareCoordinations;
if (collector.dataRange().isReversed())
comparator = comparator.reversed();
coordinations.sort(comparator);
for (Coordination c : coordinations)
{
collector.row(toStringOrNull(c.txnId()), c.kind().toString(), c.coordinationId())
.lazyCollect(columns -> {
columns.add("nodes", c, Coordination::nodes, TO_STRING)
.add("nodes_inflight", c, Coordination::inflight, TO_STRING)
.add("nodes_contacted", c, Coordination::contacted, TO_STRING)
.add("description", c, Coordination::describe, TO_STRING)
.add("participants", c, Coordination::scope, TO_STRING)
.add("replies", c, Coordination::replies, CoordinationsTable::summarise)
.add("tracker", c, Coordination::tracker, AbstractTracker::summariseTracker);
});
collector.row(toStringOrNull(c.txnId()), c.coordinationId(), c.kind().toString())
.lazyCollect(columns -> {
columns.add("nodes", c, Coordination::nodes, TO_STRING)
.add("nodes_inflight", c, Coordination::inflight, TO_STRING)
.add("nodes_contacted", c, Coordination::contacted, TO_STRING)
.add("description", c, Coordination::describe, TO_STRING)
.add("participants", c, Coordination::scope, TO_STRING)
.add("replies", c, Coordination::replies, CoordinationsTable::summarise)
.add("tracker", c, Coordination::tracker, AbstractTracker::summariseTracker);
});
}
}
private static int compareCoordinations(Coordination c1, Coordination c2)
{
int cmp = c1.txnId().compareTo(c2.txnId());
if (cmp == 0) cmp = Long.compare(c1.coordinationId(), c2.coordinationId());
return cmp;
}
private static String summarise(SortedListMap<Node.Id, ?> replies)
{
return AbstractCoordination.summariseReplies(replies, 60);
@ -382,7 +424,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
TokenKey key = TokenKey.parse((String) partitionKey[0], DatabaseDescriptor.getPartitioner());
List<Entry> cfks = new CopyOnWriteArrayList<>();
CommandStores commandStores = AccordService.instance().node().commandStores();
CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
AccordService.getBlocking(commandStores.forEach("commands_for_key table query", RoutingKeys.of(key), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> {
SafeCommandsForKey safeCfk = safeStore.get(key);
CommandsForKey cfk = safeCfk.current();
@ -530,7 +572,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
ShardDurability.ImmutableView view = ((AccordService) AccordService.instance()).shardDurability();
ShardDurability.ImmutableView view = ((AccordService) AccordService.unsafeInstance()).shardDurability();
while (view.advance())
{
@ -578,7 +620,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
DurableBefore durableBefore = AccordService.instance().node().durableBefore();
DurableBefore durableBefore = AccordService.unsafeInstance().node().durableBefore();
durableBefore.foldlWithBounds(
(entry, ignore, start, end) -> {
TableId tableId = (TableId) start.prefix();
@ -612,7 +654,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
AccordCommandStores stores = (AccordCommandStores) AccordService.instance().node().commandStores();
AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores();
for (AccordExecutor executor : stores.executors())
{
try (AccordExecutor.ExclusiveGlobalCaches cache = executor.lockCaches())
@ -634,6 +676,99 @@ public class AccordDebugKeyspace extends VirtualKeyspace
}
}
public static final class ListenersDepsTable extends AbstractLazyVirtualTable
{
private ListenersDepsTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, LISTENERS_DEPS,
"Accord Txn Dependency Listeners",
"CREATE TABLE %s (\n" +
" command_store_id int,\n" +
" waiting_on 'TxnIdUtf8Type',\n" +
" waiting_until text,\n" +
" waiter 'TxnIdUtf8Type',\n" +
" PRIMARY KEY (command_store_id, waiting_on, waiting_until, waiter)" +
')', Int32Type.instance), FAIL, UNSORTED, ASC);
}
@Override
public void collect(PartitionsCollector collector)
{
AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores();
Object[] pk = collector.singlePartitionKey();
if (pk != null)
{
CommandStore commandStore = stores.forId((Integer)pk[0]);
if (commandStore != null)
addPartition(commandStore, collector);
return;
}
stores.forAllUnsafe(commandStore -> addPartition(commandStore, collector));
}
private void addPartition(CommandStore commandStore, PartitionsCollector collector)
{
collector.partition(commandStore.id())
.collect(rows -> {
// TODO (desired): support maybe execute immediately with safeStore
AccordService.getBlocking(commandStore.chain((PreLoadContext.Empty) metadata::toString, safeStore -> { addRows(safeStore, rows); }));
});
}
private void addRows(SafeCommandStore safeStore, RowsCollector rows)
{
LocalListeners listeners = safeStore.commandStore().unsafeGetListeners();
for (LocalListeners.TxnListener listener : listeners.txnListeners())
{
rows.add(listener.waitingOn.toString(), listener.awaitingStatus.name(), listener.waiter.toString())
.eagerCollect(ignore -> {});
}
}
}
public static final class ListenersLocalTable extends AbstractLazyVirtualTable
{
private ListenersLocalTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, LISTENERS_LOCAL,
"Accord Local Listeners",
"CREATE TABLE %s (\n" +
" command_store_id int,\n" +
" waiting_on 'TxnIdUtf8Type',\n" +
" waiter text,\n" +
" PRIMARY KEY (command_store_id, waiting_on, waiter)" +
')', Int32Type.instance), FAIL, UNSORTED, ASC);
}
@Override
public void collect(PartitionsCollector collector)
{
AccordCommandStores stores = (AccordCommandStores) AccordService.unsafeInstance().node().commandStores();
Object[] pk = collector.singlePartitionKey();
if (pk != null)
{
CommandStore commandStore = stores.forId((Integer)pk[0]);
if (commandStore != null)
addPartition(commandStore, collector);
return;
}
stores.forAllUnsafe(commandStore -> addPartition(commandStore, collector));
}
private void addPartition(CommandStore commandStore, PartitionsCollector collector)
{
collector.partition(commandStore.id())
.collect(rows -> {
// TODO (desired): support maybe execute immediately with safeStore
LocalListeners listeners = commandStore.unsafeGetListeners();
for (LocalListeners.Registered listener : listeners.complexListeners())
rows.add(listener.waitingOn().toString(), listener.waiting().toString())
.eagerCollect(ignore -> {});
});
}
}
public static final class MaxConflictsTable extends AbstractLazyVirtualTable
{
private MaxConflictsTable()
@ -653,7 +788,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
CommandStores commandStores = AccordService.instance().node().commandStores();
CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
for (CommandStore commandStore : commandStores.all())
{
@ -800,7 +935,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
CommandStores commandStores = AccordService.instance().node().commandStores();
CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
for (CommandStore commandStore : commandStores.all())
{
DefaultProgressLog.ImmutableView view = ((DefaultProgressLog) commandStore.unsafeProgressLog()).immutableView();
@ -874,7 +1009,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
public void collect(PartitionsCollector collector)
{
CommandStores commandStores = AccordService.instance().node().commandStores();
CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
for (CommandStore commandStore : commandStores.all())
{
@ -929,7 +1064,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
@Override
protected void collect(PartitionsCollector collector)
{
CommandStores commandStores = AccordService.instance().node().commandStores();
CommandStores commandStores = AccordService.unsafeInstance().node().commandStores();
for (CommandStore commandStore : commandStores.all())
{
RejectBefore rejectBefore = commandStore.unsafeGetRejectBefore();
@ -1399,7 +1534,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
{
AccordService accord;
{
IAccordService iaccord = AccordService.instance();
IAccordService iaccord = AccordService.unsafeInstance();
if (!iaccord.isEnabled())
return;
@ -1599,7 +1734,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
public static final class TxnOpsTable extends AbstractMutableLazyVirtualTable
{
// TODO (expected): test each of these operations
enum Op
enum TxnOp
{
LOCALLY_ERASE_VESTIGIAL("USE WITH CAUTION: Move the command to the vestigial status, erasing its contents. This has distributed state machine implications."),
LOCALLY_INVALIDATE("USE WITH CAUTION: Move the command to the invalidated status, erasing its contents. This has distributed state machine implications."),
@ -1612,7 +1747,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
final String description;
Op(String description)
TxnOp(String description)
{
this.description = description;
}
@ -1641,7 +1776,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
TxnId txnId = TxnId.parse((String) partitionKeys[0]);
int commandStoreId = (Integer) clusteringKeys[0];
Op op = tryParse(values[0], true, Op.class, Op::valueOf);
TxnOp op = tryParse(values[0], true, TxnOp.class, TxnOp::valueOf);
switch (op)
{
@ -1655,7 +1790,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
case TRY_EXECUTE:
run(txnId, commandStoreId, safeStore -> {
SafeCommand safeCommand = safeStore.unsafeGet(txnId);
Commands.maybeExecute(safeStore, safeCommand, true, true);
Commands.maybeExecute(safeStore, safeCommand, safeCommand.current(), true, true, NotifyWaitingOnPlus.adapter(null, true, true));
return AsyncChains.success(null);
});
break;
@ -1703,7 +1838,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
Command command = safeCommand.current();
if (command == null)
throw new InvalidRequestException(txnId + " not known");
Node node = AccordService.instance().node();
Node node = AccordService.unsafeInstance().node();
AsyncResult.Settable<Void> result = new AsyncResults.SettableResult<>();
BiConsumer<Route<?>, AsyncResult.Settable<Void>> consumer = apply.apply(command);
if (command.route() == null)
@ -1723,7 +1858,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private void fetch(TxnId txnId, Timestamp executeAtIfKnown, Route<?> route, AsyncResult.Settable<Void> result)
{
Node node = AccordService.instance().node();
Node node = AccordService.unsafeInstance().node();
FetchData.fetchSpecific(Known.Apply, node, txnId, executeAtIfKnown, route, route.withHomeKey(), LatentStoreSelector.standard(), (success, fail) -> {
if (fail != null) result.setFailure(fail);
else result.setSuccess(null);
@ -1732,7 +1867,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private void recover(TxnId txnId, @Nullable Route<?> route, AsyncResult.Settable<Void> result)
{
Node node = AccordService.instance().node();
Node node = AccordService.unsafeInstance().node();
if (Route.isFullRoute(route))
{
PrepareRecovery.recover(node, node.someSequentialExecutor(), txnId, NotKnownToBeInvalid, (FullRoute<?>) route, null, LatentStoreSelector.standard(), (success, fail) -> {
@ -1742,7 +1877,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
}
else
{
MaybeRecover.maybeRecover(node, txnId, NotKnownToBeInvalid, route, ProgressToken.NONE, LatentStoreSelector.standard(), (success, fail) -> {
MaybeRecover.maybeRecover(node, txnId, NotKnownToBeInvalid, route, ProgressToken.NONE, true, LatentStoreSelector.standard(), (success, fail) -> {
if (fail != null) result.setFailure(fail);
else result.setSuccess(null);
});
@ -1751,7 +1886,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private void run(TxnId txnId, int commandStoreId, Function<SafeCommandStore, AsyncChain<Void>> apply)
{
AccordService accord = (AccordService) AccordService.instance();
AccordService accord = (AccordService) AccordService.unsafeInstance();
AccordService.getBlocking(accord.node()
.commandStores()
.forId(commandStoreId)
@ -1770,11 +1905,176 @@ public class AccordDebugKeyspace extends VirtualKeyspace
}
}
/**
* Write-only virtual table for updating Accord node states
*/
public static final class NodeOpsTable extends AbstractMutableLazyVirtualTable
{
// TODO (expected): test each of these operations
enum NodeOp
{
MARK_STALE("Mark the node as offline for an indeterminate period. Peers will be allowed to garbage collect without involving the marked node, and when it comes online it will need to rejoin the transaction log."),
UNMARK_STALE("Peers will no longer be allowed to garbage collect without involving the node."),
MARK_HARD_REMOVED("EMERGENCY USE ONLY: Mark the node as PERMANENTLY offline. It must be guaranteed not to respond at any future date. If a quorum is marked HARD_REMOVED, the remaining replicas in a shard may collectively make progress, though consistency violations are possible."),
FORCE_MARK_HARD_REMOVED("EMERGENCY USE ONLY: Mark the node as PERMANENTLY offline WHETHER OR NOT it has left the most recent topology. It must be guaranteed not to respond at any future date. If a quorum is marked HARD_REMOVED, the remaining replicas in a shard may collectively make progress, though consistency violations are possible."),
;
final String description;
NodeOp(String description)
{
this.description = description;
}
}
private NodeOpsTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, NODE_OPS,
"Update Accord Node State",
"CREATE TABLE %s (\n" +
" node_id int,\n" +
" op text," +
" PRIMARY KEY (node_id)" +
')', Int32Type.instance), FAIL, UNSORTED);
}
@Override
protected void collect(PartitionsCollector collector)
{
throw new UnsupportedOperationException(NODE_OPS + " is a write-only table");
}
@Override
protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, ColumnMetadata[] columns, Object[] values)
{
NodeId nodeId = new NodeId((Integer) partitionKeys[0]);
Set<NodeId> nodeIds = Collections.singleton(nodeId);
NodeOp op = tryParse(values[0], true, NodeOp.class, NodeOp::valueOf);
switch (op)
{
default: throw new UnhandledEnum(op);
case MARK_STALE:
AccordOperations.instance.accordMarkStale(nodeIds);
break;
case UNMARK_STALE:
AccordOperations.instance.accordMarkRejoining(nodeIds);
break;
case MARK_HARD_REMOVED:
AccordOperations.instance.accordMarkHardRemoved(nodeIds, false);
break;
case FORCE_MARK_HARD_REMOVED:
AccordOperations.instance.accordMarkHardRemoved(nodeIds, true);
break;
}
}
}
/**
* Write-only virtual table for updating Accord node states
*/
public static final class CommandStoreOpsTable extends AbstractMutableLazyVirtualTable
{
// TODO (expected): test each of these operations
enum CommandStoreOp
{
SET_PROGRESS_LOG_MODE("Set the specified progress log mode."),
UNSET_PROGRESS_LOG_MODE("Unset the specified progress log mode."),
TRY_EXECUTE_LISTENING("Try to execute all of the transactions (and their dependencies) that have registered listeners on other transactions."),
;
final String description;
CommandStoreOp(String description)
{
this.description = description;
}
}
private CommandStoreOpsTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, COMMAND_STORE_OPS,
"Update Accord CommandStore State",
"CREATE TABLE %s (\n" +
" command_store_id int,\n" +
" op text," +
" param text," +
" PRIMARY KEY (command_store_id)" +
')', Int32Type.instance), FAIL, UNSORTED);
}
@Override
protected void collect(PartitionsCollector collector)
{
throw new UnsupportedOperationException(COMMAND_STORE_OPS + " is a write-only table");
}
@Override
protected void applyRowUpdate(Object[] partitionKeys, Object[] clusteringKeys, ColumnMetadata[] columns, Object[] values)
{
int commandStoreId = (Integer) partitionKeys[0];
CommandStoreOp op = null;
String param = null;
for (int i = 0 ; i < columns.length ; ++i)
{
switch (columns[i].name.toString())
{
default: throw new IllegalArgumentException("Unknown column " + columns[i].name.toString());
case "op":
op = tryParse(values[i], true, CommandStoreOp.class, CommandStoreOp::valueOf);
break;
case "param":
param = (String) values[i];
break;
}
}
if (op == null)
throw new IllegalArgumentException("Must specify 'op'");
final Function<CommandStore, AsyncResult<?>> function;
switch (op)
{
default: throw new UnhandledEnum(op);
case SET_PROGRESS_LOG_MODE:
case UNSET_PROGRESS_LOG_MODE:
if (param == null)
throw new IllegalArgumentException("Must specify 'param' for " + op);
ModeFlag mode = tryParse(param, true, ModeFlag.class, ModeFlag::valueOf);
boolean set = op == CommandStoreOp.SET_PROGRESS_LOG_MODE;
function = commandStore -> {
DefaultProgressLog progressLog = ((DefaultProgressLog)commandStore.unsafeProgressLog());
if (set) progressLog.setMode(mode);
else progressLog.unsetMode(mode);
return AsyncResults.success(null);
};
break;
case TRY_EXECUTE_LISTENING:
if (param != null)
throw new IllegalArgumentException("'param' is not supported for " + op);
function = CommandStore::operatorTryToExecuteListeningTxns;
break;
}
AsyncResult<?> result;
if (commandStoreId < 0)
{
List<AsyncResult<?>> results = new ArrayList<>();
AccordService.unsafeInstance().node()
.commandStores()
.forAllUnsafe(commandStore -> results.add(function.apply(commandStore)));
result = AsyncResults.allOf(results);
}
else
{
result = function.apply(AccordService.unsafeInstance().node().commandStores().forId(commandStoreId));
}
AccordService.getBlocking(result);
}
}
public static class TxnBlockedByTable extends AbstractLazyVirtualTable
{
enum Reason
{Self, Txn, Key}
protected TxnBlockedByTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, TXN_BLOCKED_BY,
@ -1806,7 +2106,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
partition.collect(rows -> {
try
{
DebugBlockedTxns.visit(AccordService.instance(), txnId, maxDepth, collector.deadlineNanos(), txn -> {
DebugBlockedTxns.visit(AccordService.unsafeInstance(), txnId, maxDepth, collector.deadlineNanos(), txn -> {
String keyStr = txn.blockedViaKey == null ? "" : txn.blockedViaKey.toString();
String txnIdStr = txn.txnId == null || txn.txnId.equals(txnId) ? "" : txn.txnId.toString();
rows.add(txn.commandStoreId, txn.depth, keyStr, txnIdStr)
@ -1824,6 +2124,128 @@ public class AccordDebugKeyspace extends VirtualKeyspace
}
}
// TODO (desired): epoch/token filtering
// TODO (expected): report command_store_ids for each shard
public static final class ShardEpochsTable extends AbstractLazyVirtualTable
{
static class ShardAndEpochs
{
final long startEpoch;
final Shard shard;
long endEpoch;
ShardAndEpochs(long startEpoch, Shard shard)
{
this.startEpoch = this.endEpoch = startEpoch;
this.shard = shard;
}
}
private ShardEpochsTable()
{
super(parse(VIRTUAL_ACCORD_DEBUG, SHARD_EPOCHS,
"Accord per-CommandStore Shard Epochs",
"CREATE TABLE %s (\n" +
" table_id text,\n" +
" token_start 'TokenUtf8Type',\n" +
" epoch_start bigint,\n" +
" epoch_end bigint,\n" +
" peers text,\n" +
" peers_fast_path text,\n" +
" peers_hard_removed text,\n" +
" quorum_simple int,\n" +
" quorum_fast int,\n" +
" quorum_fast_privileged_deps int,\n" +
" quorum_fast_privileged_nodeps int,\n" +
" token_end 'TokenUtf8Type',\n" +
" PRIMARY KEY (table_id, token_start, epoch_start)" +
')', UTF8Type.instance), FAIL, ASC);
}
@Override
public void collect(PartitionsCollector collector)
{
IAccordService service = AccordService.unsafeInstance();
List<Topology> snapshot = service.node().topology().topologySnapshot();
Map<TableId, Map<TokenKey, List<ShardAndEpochs>>> tableIdLookup = new HashMap<>();
{
TableId filterTableId = null;
Object[] pk = collector.singlePartitionKey();
if (pk != null)
filterTableId = TableId.fromString((String)pk[0]);
TableId prevTableId = null;
Map<TokenKey, List<ShardAndEpochs>> startLookup = null;
for (Topology topology : snapshot)
{
for (Shard shard : topology.shards())
{
Range range = shard.range;
TableId tableId = (TableId) range.prefix();
if (filterTableId != null && !filterTableId.equals(tableId))
continue;
if (!tableId.equals(prevTableId))
{
startLookup = tableIdLookup.computeIfAbsent(tableId, ignore -> new HashMap<>());
prevTableId = tableId;
}
List<ShardAndEpochs> shards = startLookup.computeIfAbsent((TokenKey) range.start(), ignore -> new ArrayList<>());
if (!shards.isEmpty())
{
ShardAndEpochs prev = shards.get(shards.size() - 1);
if (prev.endEpoch + 1 == topology.epoch() && prev.shard.equals(shard))
{
++prev.endEpoch;
continue;
}
}
shards.add(new ShardAndEpochs(topology.epoch(), shard));
}
}
}
List<TableId> tableIds = new ArrayList<>(tableIdLookup.keySet());
tableIds.sort(Comparator.comparing(TableId::toString));
List<TokenKey> startKeys = new ArrayList<>();
for (TableId tableId : tableIds)
{
collector.partition(tableId.toString()).collect(rows -> {
startKeys.clear();
Map<TokenKey, List<ShardAndEpochs>> startLookup = tableIdLookup.get(tableId);
startKeys.addAll(startLookup.keySet());
startKeys.sort(TokenKey::compareTo);
for (TokenKey start : startKeys)
{
for (ShardAndEpochs shardsAndEpochs : startLookup.get(start))
{
Shard shard = shardsAndEpochs.shard;
rows.add(start.token().toString(), shardsAndEpochs.startEpoch)
.lazyCollect(columns -> {
columns.add("epoch_end", shardsAndEpochs.endEpoch)
.add("peers", shard.nodes, TO_STRING)
.add("peers_fast_path", shard.notInFastPath, shard.nodes::without, TO_STRING)
.add("peers_hard_removed", shard.hardRemoved, TO_STRING)
.add("quorum_simple", (int)shard.slowQuorumSize)
.add("quorum_fast", (int)shard.simpleFastQuorumSize)
.add("quorum_fast_privileged_deps", (int)shard.privilegedWithDepsFastQuorumSize)
.add("quorum_fast_privileged_nodeps", (int)shard.privilegedWithoutDepsFastQuorumSize)
.add("token_end", shard.range.end().suffix(), TO_STRING);
});
}
}
});
}
}
}
private static String printToken(RoutingKey routingKey)
{
TokenKey key = (TokenKey) routingKey;
@ -1858,12 +2280,21 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private static String toStringOrNull(Object o)
{
if (o == null)
return toStringOrNull(o, Object::toString);
}
private static <V> String toStringOrNull(V v, Function<? super V, ?> ifNotNull)
{
if (v == null)
return null;
try
{
return Objects.toString(o);
Object o = ifNotNull.apply(v);
if (o == null)
return null;
return o.toString();
}
catch (Throwable t)
{
@ -1913,7 +2344,7 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private static AccordTracing tracing()
{
return ((AccordAgent)AccordService.instance().agent()).tracing();
return ((AccordAgent)AccordService.unsafeInstance().agent()).tracing();
}
private static <E extends Enum<E>> E tryParse(Object input, boolean toUpperCase, Class<E> clazz, Function<String, E> valueOf)

2
src/java/org/apache/cassandra/db/virtual/AccordDebugRemoteKeyspace.java
View File

@ -26,6 +26,6 @@ public class AccordDebugRemoteKeyspace extends RemoteToLocalVirtualKeyspace
public AccordDebugRemoteKeyspace(String name, VirtualKeyspace wrap)
{
super(name, wrap);
super(name, wrap, vt -> !vt.name().equals(AccordDebugKeyspace.NODE_OPS));
}
}

2
src/java/org/apache/cassandra/db/virtual/VirtualKeyspaceRegistry.java
View File

@ -43,7 +43,7 @@ public final class VirtualKeyspaceRegistry
VirtualKeyspace previous = virtualKeyspaces.put(keyspace.name(), keyspace);
// some tests choose to replace the keyspace, if so make sure to cleanup tables as well
if (previous != null)
previous.tables().forEach(t -> virtualTables.remove(t));
previous.tables().forEach(t -> virtualTables.remove(t.metadata().id));
keyspace.tables().forEach(t -> virtualTables.put(t.metadata().id, t));
}

8
src/java/org/apache/cassandra/locator/InetAddressAndPort.java
View File

@ -63,7 +63,7 @@ import org.apache.cassandra.utils.FastByteOperations;
* need to sometimes return a port and sometimes not.
*
*/
public final class InetAddressAndPort extends InetSocketAddress implements Comparable<InetAddressAndPort>, Serializable
public final class InetAddressAndPort extends InetSocketAddress implements Comparable<InetAddressAndPort>, Serializable, Endpoint
{
private static final long serialVersionUID = 0;
private static final Logger logger = LoggerFactory.getLogger(InetAddressAndPort.class);
@ -382,6 +382,12 @@ public final class InetAddressAndPort extends InetSocketAddress implements Compa
return defaultPort;
}
@Override
public InetAddressAndPort endpoint()
{
return this;
}
public static final class MetadataSerializer implements org.apache.cassandra.tcm.serialization.MetadataSerializer<InetAddressAndPort>
{
public static final MetadataSerializer serializer = new MetadataSerializer();

2
src/java/org/apache/cassandra/repair/RepairJob.java
View File

@ -192,7 +192,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
requireAllEndpoints = true;
}
logger.info("{} {}.{} starting accord repair, require all endpoints {}", session.previewKind.logPrefix(session.getId()), desc.keyspace, desc.columnFamily, requireAllEndpoints);
AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints);
AccordRepair repair = new AccordRepair(ctx, cfs, desc.sessionId, desc.keyspace, desc.ranges, requireAllEndpoints, allEndpoints);
return repair.repair(taskExecutor).flatMap(accordRepairResult -> {
// Propagate the HLC discovered during Accord repair to Paxos so Paxos doesn't use ballots < Accord has already used
if (accordRepairResult.maxHlc != IAccordService.NO_HLC)

7
src/java/org/apache/cassandra/service/RetryStrategy.java
View File

@ -276,6 +276,13 @@ public class RetryStrategy implements WaitStrategy
default: throw new IllegalArgumentException("Invalid modifier specification: unrecognised property '" + key + '\'');
case "retries":
retries = Integer.parseInt(value);
if (retries < 0)
throw new IllegalArgumentException("retries must be non-negative (retries=" + value + " supplied)");
break;
case "attempts":
retries = Integer.parseInt(value);
if (retries < 0)
throw new IllegalArgumentException("Must permit at least one attempt (attempts=" + value + " supplied)");
break;
case "rnd":
if (randomizer != null)

2
src/java/org/apache/cassandra/service/TimeoutStrategy.java
View File

@ -85,7 +85,7 @@ public class TimeoutStrategy implements WaitStrategy
static final Pattern WAIT = Pattern.compile(
"\\s*(?<const>0|[0-9]+[mu]?s)" +
"|\\s*((p(?<perc>[0-9]+)(\\((?<rw>r|w|rw|wr)\\))?)?|(?<constbase>0|[0-9]+[mu]?s))" +
"\\s*(([*]\\s*(?<mod>[0-9.]+))?\\s*(?<modkind>[*^]\\s*attempts)?)?\\s*");
"\\s*(([*]\\s*(?<mod>[0-9.]+))?\\s*(?<modkind>[*^]\\s*attempts?)?)?\\s*");
static final Pattern TIME = Pattern.compile(
"0|[0-9]+[mu]?s");

2
src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
View File

@ -474,7 +474,7 @@ public class AccordCommandStore extends CommandStore
@Override
protected void ensureDurable(Ranges ranges, RedundantBefore onCommandStoreDurable)
{
if (!CommandsForKey.reportLinearizabilityViolations())
if (node().isReplaying())
return;
long reportId = nextDurabilityLoggingId.incrementAndGet();

2
src/java/org/apache/cassandra/service/accord/AccordCommandStores.java
View File

@ -191,7 +191,7 @@ public class AccordCommandStores extends CommandStores implements CacheSize
return Arrays.asList(executors.clone());
}
public void waitForQuiescense()
public void waitForQuiescence()
{
for (AccordExecutor executor : this.executors)
executor.waitForQuiescence();

155
src/java/org/apache/cassandra/service/accord/AccordConfigurationService.java
View File

@ -34,13 +34,15 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import accord.api.Agent;
import accord.api.TopologySorter;
import accord.coordinate.tracking.NotifyTracker;
import accord.coordinate.tracking.RequestStatus;
import accord.impl.AbstractConfigurationService;
import accord.local.Node;
import accord.primitives.Ranges;
import accord.topology.Shard;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.utils.Invariants;
import accord.utils.SortedListSet;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.agrona.collections.LongArrayList;
@ -48,8 +50,6 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessageDelivery;
import org.apache.cassandra.net.MessagingService;
@ -68,24 +68,25 @@ import org.slf4j.LoggerFactory;
import static org.apache.cassandra.service.accord.AccordTopology.tcmIdToAccord;
import static org.apache.cassandra.utils.Simulate.With.MONITORS;
// TODO (desired): listen to FailureDetector and rearrange fast path accordingly
@Simulate(with=MONITORS)
public class AccordConfigurationService extends AbstractConfigurationService<AccordConfigurationService.EpochState, AccordConfigurationService.EpochHistory> implements AccordEndpointMapper, AccordSyncPropagator.Listener, Shutdownable
public class AccordConfigurationService extends AbstractConfigurationService<AccordConfigurationService.EpochState, AccordConfigurationService.EpochHistory> implements AccordSyncPropagator.Listener, Shutdownable
{
public static final Logger logger = LoggerFactory.getLogger(AccordConfigurationService.class);
private final AccordSyncPropagator syncPropagator;
public final WatermarkCollector watermarkCollector;
private final WatermarkCollector watermarkCollector;
private final AccordEndpointMapper endpointMapper;
private enum State { INITIALIZED, STARTED, SHUTDOWN }
@GuardedBy("this")
private State state = State.INITIALIZED;
private volatile EndpointMapping mapping = EndpointMapping.EMPTY;
public enum SyncStatus { NOT_STARTED, NOTIFYING, COMPLETED }
static class EpochState extends AbstractConfigurationService.AbstractEpochState
{
private NotifyTracker notifyTracker;
private volatile SyncStatus syncStatus = SyncStatus.NOT_STARTED;
protected final AsyncResult.Settable<Void> localSyncNotified = AsyncResults.settable();
@ -94,11 +95,29 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
super(epoch);
}
void setSyncStatus(SyncStatus status)
void ack(Node.Id id)
{
this.syncStatus = status;
if (status == SyncStatus.COMPLETED)
localSyncNotified.trySuccess(null);
if (notifyTracker != null && RequestStatus.Success == notifyTracker.recordSuccess(id))
doneNotifying();
}
void nack(Node.Id id)
{
if (notifyTracker != null && RequestStatus.Success == notifyTracker.recordFailure(id))
doneNotifying();
}
void startNotifying()
{
syncStatus = SyncStatus.NOTIFYING;
notifyTracker = new NotifyTracker(new Topologies.Single((TopologySorter) null, topology));
}
void doneNotifying()
{
syncStatus = SyncStatus.COMPLETED;
localSyncNotified.trySuccess(null);
notifyTracker = null;
}
AsyncResult<Topology> received()
@ -140,17 +159,18 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
}
}
public AccordConfigurationService(Node.Id node, Agent agent, MessageDelivery messagingService, IFailureDetector failureDetector, ScheduledExecutorPlus scheduledTasks)
public AccordConfigurationService(Node.Id node, Agent agent, AccordEndpointMapper endpointMapper, MessageDelivery messagingService, ScheduledExecutorPlus scheduledTasks)
{
super(node, agent);
this.syncPropagator = new AccordSyncPropagator(localId, this, messagingService, failureDetector, scheduledTasks, this);
this.syncPropagator = new AccordSyncPropagator(localId, endpointMapper, messagingService, scheduledTasks, this);
this.watermarkCollector = new WatermarkCollector();
this.endpointMapper = endpointMapper;
listeners.add(watermarkCollector);
}
public AccordConfigurationService(Node.Id node, Agent agent)
{
this(node, agent, MessagingService.instance(), FailureDetector.instance, ScheduledExecutors.scheduledTasks);
this(node, agent, new EndpointMapping.Updateable(), MessagingService.instance(), ScheduledExecutors.scheduledTasks);
}
@Override
@ -166,11 +186,9 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
{
Invariants.require(state == State.INITIALIZED, "Expected state to be INITIALIZED but was %s", state);
state = State.STARTED;
// for all nodes removed, or pending removal, mark them as removed, so we don't wait on their replies
Map<Node.Id, Long> removedNodes = mapping.removedNodes();
for (Map.Entry<Node.Id, Long> e : removedNodes.entrySet())
onNodeRemoved(e.getValue(), currentTopology(), e.getKey());
endpointMapper.removedNodes().forEach((removed, removedIn) -> {
onNodeRemoved(removedIn, new Node.Id(removed.id));
});
}
@Override
@ -200,28 +218,9 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
return isTerminated();
}
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
{
return mapping.mappedIdOrNull(endpoint);
}
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
{
return mapping.mappedEndpointOrNull(id);
}
@VisibleForTesting
synchronized void updateMapping(EndpointMapping mapping)
{
if (mapping.epoch() > this.mapping.epoch())
this.mapping = mapping;
}
public synchronized void updateMapping(ClusterMetadata metadata)
{
updateMapping(AccordTopology.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory));
endpointMapper.updateMapping(metadata);
}
private void reportMetadata(ClusterMetadata metadata)
@ -234,14 +233,6 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
Topology topology = AccordTopology.createAccordTopology(metadata);
updateMapping(metadata);
if (Invariants.isParanoid())
{
for (Node.Id node : topology.nodes())
{
if (mapping.mappedEndpointOrNull(node) == null)
throw new IllegalStateException(String.format("Epoch %d has node %s but mapping does not!", topology.epoch(), node));
}
}
reportTopology(topology);
Set<Node.Id> stillLiveNodes = metadata.directory.states.entrySet()
.stream()
@ -264,24 +255,14 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
for (Node.Id removedNode : removedNodes)
{
if (topology.epoch() >= minEpoch)
onNodeRemoved(topology.epoch(), previous, removedNode);
onNodeRemoved(topology.epoch(), removedNode);
}
}
private static boolean shareShard(Topology current, Node.Id target, Node.Id self)
{
for (Shard shard : current.shards())
{
if (!shard.contains(target)) continue;
if (shard.contains(self)) return true;
}
return false;
}
public void onNodeRemoved(long epoch, Topology current, Node.Id removed)
public void onNodeRemoved(long epoch, Node.Id removed)
{
syncPropagator.onNodesRemoved(removed);
// TODO (now): it seems to be incorrect to mark remote syncs complete if/when node got removed.
// TODO (desired): this is test only; make integration cleaner
for (long oldEpoch : nonCompletedEpochsBefore(epoch))
receiveRemoteSyncCompletePreListenerNotify(removed, oldEpoch);
@ -358,7 +339,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
fetchTopologyInternal(i);
}
private static final Object Success = new Object();
private static final Object SUCCESS = new Object();
protected void fetchTopologyInternal(long epoch)
{
@ -402,7 +383,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
ClusterMetadata metadata = ClusterMetadata.current();
if (metadata.epoch.getEpoch() == epoch)
{
onResult.accept(Success, null);
onResult.accept(SUCCESS, null);
return;
}
@ -410,7 +391,7 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
peers.remove(FBUtilities.getBroadcastAddressAndPort());
if (peers.isEmpty())
{
onResult.accept(Success, null);
onResult.accept(SUCCESS, null);
return;
}
@ -420,14 +401,14 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
if (t != null)
{
if (currentEpoch() >= epoch)
onResult.accept(Success, null);
onResult.accept(SUCCESS, null);
else
onResult.accept(null, t);
}
else
{
topologyRange.forEach(this::reportTopology, epoch, 1);
onResult.accept(Success, null);
onResult.accept(SUCCESS, null);
}
});
});
@ -440,39 +421,33 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
}
@Override
public void reportTopology(Topology topology, boolean isLoad, boolean startSync)
public void reportTopology(Topology topology)
{
long tcmEpoch = ClusterMetadata.current().epoch.getEpoch();
Invariants.require(topology.epoch() <= tcmEpoch,
"Reported topology %s not known to TCM", topology.epoch(), tcmEpoch);
super.reportTopology(topology, isLoad, startSync);
super.reportTopology(topology);
}
@Override
protected void onReadyToCoordinate(Topology topology, boolean startSync)
protected void onReadyToCoordinate(Topology topology)
{
long epoch = topology.epoch();
EpochState epochState = getOrCreateEpochState(epoch);
synchronized (this)
{
if (!startSync || epochState.syncStatus != SyncStatus.NOT_STARTED)
if (epochState.syncStatus != SyncStatus.NOT_STARTED)
return;
epochState.setSyncStatus(SyncStatus.NOTIFYING);
if (topology.nodes().contains(localId)) epochState.startNotifying();
else epochState.doneNotifying();
}
Set<Node.Id> notify = SortedListSet.allOf(topology.nodes());
notify.remove(localId);
syncPropagator.reportSyncComplete(epoch, notify, localId);
syncPropagator.reportCoordinationReady(epoch, topology.nodes(), localId);
}
@Override
public synchronized void onEndpointAck(Node.Id id, long epoch)
{
}
@Override
public void onComplete(long epoch)
{
if (epochs.wasTruncated(epoch))
return;
@ -480,13 +455,21 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
EpochState epochState = getOrCreateEpochState(epoch);
synchronized (this)
{
epochState.setSyncStatus(SyncStatus.COMPLETED);
epochState.ack(id);
}
}
@Override
protected synchronized void receiveRemoteSyncCompletePreListenerNotify(Node.Id node, long epoch)
public synchronized void onEndpointNack(Node.Id id, long epoch)
{
if (epochs.wasTruncated(epoch))
return;
EpochState epochState = getOrCreateEpochState(epoch);
synchronized (this)
{
epochState.nack(id);
}
}
@Override
@ -546,6 +529,16 @@ public class AccordConfigurationService extends AbstractConfigurationService<Acc
Invariants.require(state == State.STARTED, "Expected state to be STARTED but was %s", state);
}
public WatermarkCollector watermarkCollector()
{
return watermarkCollector;
}
public AccordEndpointMapper endpointMapper()
{
return endpointMapper;
}
@VisibleForTesting
public static class EpochSnapshot
{

3
src/java/org/apache/cassandra/service/accord/AccordDataStore.java
View File

@ -26,7 +26,6 @@ import accord.local.CommandStore;
import accord.local.Node;
import accord.local.RedundantBefore;
import accord.local.SafeCommandStore;
import accord.local.cfk.CommandsForKey;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.SyncPoint;
@ -47,7 +46,7 @@ public class AccordDataStore implements DataStore
*/
public void ensureDurable(CommandStore commandStore, Ranges ranges, RedundantBefore reportOnSuccess)
{
if (!CommandsForKey.reportLinearizabilityViolations())
if (commandStore.node().isReplaying())
return;
logger.debug("{} awaiting local data durability of {}", commandStore, ranges);

25
src/java/org/apache/cassandra/service/accord/AccordEndpointMapper.java
View File

@ -18,25 +18,28 @@
package org.apache.cassandra.service.accord;
import java.util.Map;
import javax.annotation.Nullable;
import accord.local.Node;
import accord.utils.Invariants;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tcm.ClusterMetadata;
/**
* Maps network addresses to accord ids
*/
public interface AccordEndpointMapper
{
Node.Id mappedIdOrNull(InetAddressAndPort endpoint);
InetAddressAndPort mappedEndpointOrNull(Node.Id id);
default @Nullable Node.Id mappedIdOrNull(InetAddressAndPort endpoint) { return mappedIdOrNull(endpoint, null); }
@Nullable Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped);
default @Nullable InetAddressAndPort mappedEndpointOrNull(Node.Id id) { return mappedEndpointOrNull(id, null); }
@Nullable InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped);
default Node.Id mappedId(InetAddressAndPort endpoint)
{
return Invariants.nonNull(mappedIdOrNull(endpoint), "Unable to map address %s to a Node.Id", endpoint);
}
enum NodeStatus { UNKNOWN, UNHEALTHY, HEALTHY }
default InetAddressAndPort mappedEndpoint(Node.Id id)
{
return Invariants.nonNull(mappedEndpointOrNull(id), "Unable to map node id %s to a InetAddressAndPort", id);
}
default boolean isRemoved(Node.Id id) { return removedNodes().containsKey(id); }
Map<Node.Id, Long> removedNodes();
NodeStatus nodeStatus(Node.Id id);
default void updateMapping(ClusterMetadata metadata) {}
}

16
src/java/org/apache/cassandra/service/accord/AccordFastPath.java
View File

@ -25,7 +25,7 @@ import java.util.Objects;
import accord.local.Node;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.util.DataInputPlus;
@ -235,14 +235,14 @@ public class AccordFastPath implements MetadataValue<AccordFastPath>
return lastModified;
}
public ImmutableSet<Node.Id> unavailableIds()
public SortedArrayList<Node.Id> unavailableIds()
{
ImmutableSet.Builder<Node.Id> builder = ImmutableSet.builder();
info.entrySet().stream()
.filter(entry -> entry.getValue().status.isUnavailable())
.map(Map.Entry::getKey)
.forEach(builder::add);
return builder.build();
// TODO (expected): why don't we save this?
Node.Id[] ids = info.entrySet().stream()
.filter(entry -> entry.getValue().status.isUnavailable())
.map(Map.Entry::getKey)
.toArray(Node.Id[]::new);
return SortedArrayList.ofUnsorted(ids);
}
public static final MetadataSerializer<AccordFastPath> serializer = new MetadataSerializer<AccordFastPath>()

6
src/java/org/apache/cassandra/service/accord/AccordFastPathCoordinator.java
View File

@ -222,14 +222,14 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi
@Override
public void onAlive(InetAddressAndPort endpoint, EndpointState state)
{
Node.Id node = configService.mappedIdOrNull(endpoint);
Node.Id node = configService.endpointMapper().mappedIdOrNull(endpoint);
if (node != null) onAlive(node);
}
@Override
public void onDead(InetAddressAndPort endpoint, EndpointState state)
{
Node.Id node = configService.mappedIdOrNull(endpoint);
Node.Id node = configService.endpointMapper().mappedIdOrNull(endpoint);
if (node != null) onDead(node);
}
}
@ -335,7 +335,7 @@ public abstract class AccordFastPathCoordinator implements ChangeListener, Confi
}
@Override
public AsyncResult<Void> onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync)
public AsyncResult<Void> onTopologyUpdate(Topology topology)
{
updatePeers(topology);
return SUCCESS;

30
src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java
View File

@ -25,11 +25,15 @@ import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Data;
import accord.api.DataStore;
import accord.api.Query;
import accord.api.Read;
import accord.api.Update;
import accord.coordinate.tracking.AbstractTracker;
import accord.impl.AbstractFetchCoordinator;
import accord.local.CommandStore;
import accord.local.Node;
@ -80,6 +84,7 @@ import static org.apache.cassandra.utils.CollectionSerializers.serializedMapSize
public class AccordFetchCoordinator extends AbstractFetchCoordinator implements StreamManager.StreamListener
{
private static final Logger logger = LoggerFactory.getLogger(AccordFetchCoordinator.class);
private static final Query noopQuery = (txnId, executeAt, keys, data, read, update) -> null;
public static class StreamData implements Data
@ -101,7 +106,6 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
{
TimeUUID.Serializer.instance.serialize(info.planId, out);
out.writeBoolean(info.hasData);
}
public SessionInfo deserialize(DataInputPlus in) throws IOException
@ -215,19 +219,28 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
Invariants.nonNull(future);
Invariants.require(this.future == null, "future was not null: %s", this.future);
this.future = future;
logger.info("StreamFuture for plan {} received for bootstrap of {} from {}", planId, range, from);
maybeListen();
}
private void maybeListen()
{
// TODO (required): if for some reason the stream isn't initiated this hangs
if (range == null || future == null)
return;
Invariants.nonNull(from);
future.addCallback((state, fail) -> {
if (fail == null) success(from, Ranges.of(range));
else fail(from, Ranges.of(range), fail);
if (fail == null)
{
logger.info("Reporting success of plan {} for bootstrap of {} from {}", planId, range, from);
success(from, Ranges.of(range));
}
else
{
logger.info("Reporting failure of plan {} for bootstrap of {} from {}", planId, range, from, fail);
fail(from, Ranges.of(range), fail);
}
}, ((AccordCommandStore) commandStore()).taskExecutor());
}
}
@ -302,6 +315,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
StreamPlan plan = new StreamPlan(StreamOperation.BOOTSTRAP, 1, false,
null, PreviewKind.NONE).flushBeforeTransfer(true);
logger.info("StreamPlan {} created for bootstrap of {} to {}", plan.planId(), range, to);
RangesAtEndpoint ranges = RangesAtEndpoint.toDummyList(Collections.singleton(range.toKeyspaceRange()));
plan.transferRanges(to, table.keyspace, ranges, table.name);
StreamResultFuture future = plan.execute();
@ -382,6 +396,12 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
super(node, node.someSequentialExecutor(), ranges, syncPoint, fetchRanges, commandStore);
}
@Override
public AbstractTracker<?> tracker()
{
return null;
}
@Override
public void start()
{
@ -424,10 +444,12 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
streamData.streams.forEach((range, streamInfo) -> {
if (streamInfo.hasData)
{
logger.info("StreamPlan {} created for bootstrap of {} from {}", streamInfo.planId, range, from);
stream(streamInfo.planId).rangeReceived(range, from);
}
else
{
logger.info("StreamPlan {} created for bootstrap of {} from {} with no data to stream; succeeding immediately.", streamInfo.planId, range, from);
// if there was no data to stream, no connection is initiated, and we aren't notified via the stream
// listener, so the stream initiator notifies us and we mark it complete here
success(from, Ranges.of(range));

6
src/java/org/apache/cassandra/service/accord/AccordJournal.java
View File

@ -50,7 +50,6 @@ import accord.local.Node;
import accord.local.RedundantBefore;
import accord.primitives.EpochSupplier;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.SaveStatus;
@ -652,7 +651,7 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
if (segments != null && route != null)
{
for (long segment : segments)
journalTable.safeNotify(index -> index.update(segment, key.commandStoreId, txnId, (Route)route));
journalTable.safeNotify(index -> index.update(segment, key.commandStoreId, txnId, (Route<?>) route));
}
return null;
}).begin((success, fail) -> {
@ -1069,8 +1068,7 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
break;
case PARTIAL_TXN:
Invariants.require(partialTxn != null, "%s", this);
if (partialTxn instanceof ByteBuffer) out.write(((ByteBuffer) partialTxn).duplicate());
else CommandSerializers.partialTxn.serialize((PartialTxn) partialTxn, out, userVersion);
CommandSerializers.partialTxn.serialize(partialTxn, out, userVersion);
break;
case PARTIAL_DEPS:
Invariants.require(partialDeps != null, "%s", this);

41
src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
View File

@ -163,7 +163,7 @@ public class AccordMessageSink implements MessageSink
this.callbacks = callbacks;
}
public AccordMessageSink(AccordAgent agent, AccordConfigurationService endpointMapper, RequestCallbacks callbacks)
public AccordMessageSink(AccordAgent agent, AccordEndpointMapper endpointMapper, RequestCallbacks callbacks)
{
this(agent, MessagingService.instance(), endpointMapper, callbacks);
}
@ -174,7 +174,9 @@ public class AccordMessageSink implements MessageSink
Verb verb = VerbMapping.getVerb(request);
Preconditions.checkNotNull(verb, "Verb is null for type %s", request.type());
Message<Request> message = Message.out(verb, request);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to, message);
if (endpoint == null)
return;
logger.trace("Sending {} {} to {}", verb, message.payload, endpoint);
messaging.send(message, endpoint);
}
@ -213,7 +215,13 @@ public class AccordMessageSink implements MessageSink
}
Message<Request> message = Message.out(verb, request, expiresAtNanos);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(to);
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to, message);
if (endpoint == null)
{
executor.execute(() -> callback.onFailure(to, null));
return null;
}
logger.trace("Sending {} {} to {}", verb, message.payload, endpoint);
Cancellable cancellable = callbacks.registerAt(message.id(), executor, callback, to, nowNanos, slowAtNanos, expiresAtNanos, NANOSECONDS);
messaging.send(message, endpoint);
@ -221,26 +229,31 @@ public class AccordMessageSink implements MessageSink
}
@Override
public void reply(Node.Id replyingToNode, ReplyContext replyContext, Reply reply)
public void reply(Node.Id replyingTo, ReplyContext replyContext, Reply reply)
{
ResponseContext respondTo = (ResponseContext) replyContext;
Message<?> responseMsg = Message.responseWith(reply, respondTo);
Message<?> message = Message.responseWith(reply, respondTo);
if (!reply.isFinal())
responseMsg = responseMsg.withFlag(MessageFlag.NOT_FINAL);
message = message.withFlag(MessageFlag.NOT_FINAL);
checkReplyType(reply, respondTo);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode);
logger.trace("Replying {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
messaging.send(responseMsg, endpoint);
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(replyingTo, message);
if (endpoint == null)
return;
logger.trace("Replying {} {} to {}", message.verb(), message.payload, endpoint);
messaging.send(message, endpoint);
}
@Override
public void replyWithUnknownFailure(Node.Id replyingToNode, ReplyContext replyContext, Throwable failure)
public void replyWithUnknownFailure(Node.Id replyingTo, ReplyContext replyContext, Throwable failure)
{
ResponseContext respondTo = (ResponseContext) replyContext;
Message<?> responseMsg = Message.failureResponse(RequestFailureReason.UNKNOWN, failure, respondTo);
InetAddressAndPort endpoint = endpointMapper.mappedEndpoint(replyingToNode);
logger.trace("Replying with failure {} {} to {}", responseMsg.verb(), responseMsg.payload, endpoint);
messaging.send(responseMsg, endpoint);
Message<?> message = Message.failureResponse(RequestFailureReason.UNKNOWN, failure, respondTo);
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(replyingTo, message);
if (endpoint == null)
return;
logger.trace("Replying with failure {} {} to {}", message.verb(), message.payload, endpoint);
messaging.send(message, endpoint);
}
private static void checkReplyType(Reply reply, ResponseContext respondTo)

39
src/java/org/apache/cassandra/service/accord/AccordOperations.java
View File

@ -27,6 +27,7 @@ import java.util.stream.Collectors;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.transformations.AccordMarkHardRemoved;
import org.apache.cassandra.tcm.transformations.AccordMarkStale;
import org.apache.cassandra.tcm.transformations.AccordMarkRejoining;
import org.apache.cassandra.utils.MBeanWrapper;
@ -55,7 +56,7 @@ public class AccordOperations implements AccordOperationsMBean
ClusterMetadata metadata = ClusterMetadata.current();
info.put("EPOCH", Long.toString(metadata.epoch.getEpoch()));
String staleReplicas = metadata.accordStaleReplicas.ids().stream().sorted().map(Object::toString).collect(Collectors.joining(","));
String staleReplicas = metadata.accordStaleReplicas.stale().stream().sorted().map(Object::toString).collect(Collectors.joining(","));
info.put("STALE_REPLICAS", staleReplicas);
return info;
}
@ -63,14 +64,44 @@ public class AccordOperations implements AccordOperationsMBean
@Override
public void accordMarkStale(List<String> nodeIdStrings)
{
Set<NodeId> nodeIds = nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet());
cms.commit(new AccordMarkStale(nodeIds));
accordMarkStale(parse(nodeIdStrings));
}
@Override
public void accordMarkHardRemoved(List<String> nodeIdStrings)
{
accordMarkHardRemoved(parse(nodeIdStrings), false);
}
@Override
public void accordForceMarkHardRemoved(List<String> nodeIdStrings)
{
accordMarkHardRemoved(parse(nodeIdStrings), true);
}
@Override
public void accordMarkRejoining(List<String> nodeIdStrings)
{
Set<NodeId> nodeIds = nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet());
accordMarkRejoining(parse(nodeIdStrings));
}
private static Set<NodeId> parse(List<String> nodeIdStrings)
{
return nodeIdStrings.stream().map(NodeId::fromString).collect(Collectors.toSet());
}
public void accordMarkStale(Set<NodeId> nodeIds)
{
cms.commit(new AccordMarkStale(nodeIds));
}
public void accordMarkHardRemoved(Set<NodeId> nodeIds, boolean force)
{
cms.commit(new AccordMarkHardRemoved(nodeIds, force));
}
public void accordMarkRejoining(Set<NodeId> nodeIds)
{
cms.commit(new AccordMarkRejoining(nodeIds));
}
}

4
src/java/org/apache/cassandra/service/accord/AccordOperationsMBean.java
View File

@ -27,5 +27,9 @@ public interface AccordOperationsMBean
void accordMarkStale(List<String> nodeIds);
void accordMarkHardRemoved(List<String> nodeIds);
void accordForceMarkHardRemoved(List<String> nodeIds);
void accordMarkRejoining(List<String> nodeIds);
}

8
src/java/org/apache/cassandra/service/accord/AccordResponseVerbHandler.java
View File

@ -60,7 +60,13 @@ class AccordResponseVerbHandler<T extends Reply> implements IVerbHandler<T>
return;
}
Node.Id from = endpointMapper.mappedId(message.from());
Node.Id from = endpointMapper.mappedIdOrNull(message.from(), message);
if (from == null)
{
dropping.debug(message.verb(), message.from());
return;
}
logger.trace("Receiving {} from {}", message.payload, message.from());
if (message.isFailureResponse())
{

4
src/java/org/apache/cassandra/service/accord/AccordResult.java
View File

@ -172,6 +172,10 @@ public class AccordResult<V> extends AsyncFuture<V> implements BiConsumer<V, Thr
trySuccess((V) RetryWithNewProtocolResult.instance);
return false;
}
else if (fail instanceof TimeoutException)
{
report = bookkeeping.newTimeout(txnId, keysOrRanges);
}
else
{
report = bookkeeping.newFailed(txnId, keysOrRanges);

126
src/java/org/apache/cassandra/service/accord/AccordService.java
View File

@ -63,7 +63,6 @@ import accord.local.Node;
import accord.local.Node.Id;
import accord.local.ShardDistributor.EvenSplit;
import accord.local.UniqueTimeService.AtomicUniqueTimeWithStaleReservation;
import accord.local.cfk.CommandsForKey;
import accord.local.durability.DurabilityService;
import accord.local.durability.ShardDurability;
import accord.messages.Reply;
@ -236,17 +235,18 @@ public class AccordService implements IAccordService, Shutdownable
// TODO (expected): wrap this in an inner class that is statically initialised and final
// tests can specify a DelegatingService if they want to override
private static IAccordService instance;
private static IAccordService unsafeInstance;
@VisibleForTesting
public static void unsafeSetNewAccordService(IAccordService service)
{
instance = service;
unsafeInstance = instance = service;
}
@VisibleForTesting
public static void unsafeSetNoop()
{
instance = NOOP_SERVICE;
unsafeInstance = instance = NOOP_SERVICE;
}
public static boolean isSetup()
@ -258,7 +258,7 @@ public class AccordService implements IAccordService, Shutdownable
{
if (!isSetup()) return ignore -> {};
AccordService i = (AccordService) instance();
return i.configService().watermarkCollector.handler;
return i.configService().watermarkCollector().handler;
}
public static IVerbHandler<? extends Request> requestHandlerOrNoop()
@ -278,7 +278,7 @@ public class AccordService implements IAccordService, Shutdownable
{
if (!DatabaseDescriptor.getAccordTransactionsEnabled())
{
instance = NOOP_SERVICE;
unsafeInstance = instance = NOOP_SERVICE;
return null;
}
@ -286,8 +286,17 @@ public class AccordService implements IAccordService, Shutdownable
return (AccordService) instance;
AccordService as = new AccordService(AccordTopology.tcmIdToAccord(tcmId));
as.startup();
replayJournal(as);
unsafeInstance = as;
as.node.unsafeSetReplaying(true);
try
{
as.startup();
replayJournal(as);
}
finally
{
as.node.unsafeSetReplaying(false);
}
as.finishInitialization();
@ -320,23 +329,15 @@ public class AccordService implements IAccordService, Shutdownable
{
logger.info("Starting journal replay.");
long before = Clock.Global.nanoTime();
CommandsForKey.disableLinearizabilityViolationsReporting();
try
{
if (as.journalConfiguration().replayMode() == RESET)
AccordKeyspace.truncateCommandsForKey();
if (as.journalConfiguration().replayMode() == RESET)
AccordKeyspace.truncateCommandsForKey();
as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop());
as.journal().replay(as.node().commandStores());
logger.info("Waiting for command stores to quiesce.");
((AccordCommandStores)as.node.commandStores()).waitForQuiescense();
as.journal.unsafeSetStarted();
as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start());
}
finally
{
CommandsForKey.enableLinearizabilityViolationsReporting();
}
as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().stop());
as.journal().replay(as.node().commandStores());
logger.info("Waiting for command stores to quiesce.");
((AccordCommandStores)as.node.commandStores()).waitForQuiescence();
as.journal.unsafeSetStarted();
as.node.commandStores().forAllUnsafe(cs -> cs.unsafeProgressLog().start());
long after = Clock.Global.nanoTime();
logger.info("Finished journal replay. {}ms elapsed", NANOSECONDS.toMillis(after - before));
@ -351,13 +352,18 @@ public class AccordService implements IAccordService, Shutdownable
public static IAccordService instance()
{
if (!DatabaseDescriptor.getAccordTransactionsEnabled())
return NOOP_SERVICE;
IAccordService i = instance;
Invariants.require(i != null, "AccordService was not started");
return i;
}
public static IAccordService unsafeInstance()
{
IAccordService i = unsafeInstance;
Invariants.require(i != null, "AccordService was not started");
return i;
}
public static boolean started()
{
if (!DatabaseDescriptor.getAccordTransactionsEnabled())
@ -379,7 +385,7 @@ public class AccordService implements IAccordService, Shutdownable
this.journal = new AccordJournal(DatabaseDescriptor.getAccord().journal);
this.configService = new AccordConfigurationService(localId, agent);
this.fastPathCoordinator = AccordFastPathCoordinator.create(localId, configService);
this.messageSink = new AccordMessageSink(agent, configService, callbacks);
this.messageSink = new AccordMessageSink(agent, configService.endpointMapper(), callbacks);
this.node = new Node(localId,
messageSink,
configService,
@ -390,18 +396,18 @@ public class AccordService implements IAccordService, Shutdownable
new DefaultRandom(),
scheduler,
CompositeTopologySorter.create(SizeOfIntersectionSorter.SUPPLIER,
new AccordTopologySorter.Supplier(configService, DatabaseDescriptor.getNodeProximity())),
new AccordTopologySorter.Supplier(configService.endpointMapper(), DatabaseDescriptor.getNodeProximity())),
DefaultRemoteListeners::new,
ignore -> callbacks,
DefaultProgressLogs::new,
DefaultLocalListeners.Factory::new,
AccordCommandStores.factory(),
new AccordInteropFactory(configService),
new AccordInteropFactory(configService.endpointMapper()),
journal.durableBeforePersister(),
journal);
this.nodeShutdown = toShutdownable(node);
this.requestHandler = new AccordVerbHandler<>(node, configService);
this.responseHandler = new AccordResponseVerbHandler<>(callbacks, configService);
this.requestHandler = new AccordVerbHandler<>(node, configService.endpointMapper());
this.responseHandler = new AccordResponseVerbHandler<>(callbacks, configService.endpointMapper());
}
@Override
@ -416,34 +422,21 @@ public class AccordService implements IAccordService, Shutdownable
configService.updateMapping(metadata);
List<TopologyUpdate> images = journal.replayTopologies();
// Instantiate latest topology from the log, if known
if (!images.isEmpty())
node.commandStores().initializeTopologyUnsafe(images.get(images.size() - 1));
// Replay local epochs
for (TopologyUpdate image : images)
configService.reportTopology(image.global);
// Subscribe to TCM events
ChangeListener prevListener = MetadataChangeListener.instance.collector.getAndSet(new ChangeListener()
{
@Override
public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
// Initialise command stores using latest topology from the log;
// if there are no local command stores, don't report any topologies and simply fetch the latest known in the cluster
// this avoids a registered (not joined) node learning of topologies, then later restarting with some intervening
// epochs having been garbage collected by the other nodes in the cluster
TopologyUpdate last = images.get(images.size() - 1);
if (!last.commandStores.isEmpty())
{
if (state != State.SHUTDOWN)
configService.maybeReportMetadata(next);
node.commandStores().initializeTopologyUnsafe(last);
// Replay local epochs
for (TopologyUpdate image : images)
configService.reportTopology(image.global);
}
});
Invariants.require((prevListener instanceof MetadataChangeListener.PreInitStateCollector),
"Listener should have been initialized with Accord pre-init state collector, but was " + prevListener.getClass());
MetadataChangeListener.PreInitStateCollector preinit = (MetadataChangeListener.PreInitStateCollector) prevListener;
for (ClusterMetadata item : preinit.getItems())
{
if (item.epoch.getEpoch() > Epoch.FIRST.getEpoch())
configService.maybeReportMetadata(item);
}
}
@ -464,7 +457,32 @@ public class AccordService implements IAccordService, Shutdownable
TopologyRange remote = fetchTopologies(highestKnown + 1);
if (remote != null) // TODO (required): if remote.min > highestKnown + 1, should we decide if we need to truncate our local topologies? Probably not until startup has finished.
{
remote.forEach(configService::reportTopology, remote.min, Integer.MAX_VALUE);
if (remote.current > highestKnown)
highestKnown = remote.current;
}
// Subscribe to TCM events, and collect any we may have missed to report now
ChangeListener prevListener = MetadataChangeListener.instance.collector.getAndSet(new ChangeListener()
{
@Override
public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
{
if (state != State.SHUTDOWN)
configService.maybeReportMetadata(next);
}
});
Invariants.require((prevListener instanceof MetadataChangeListener.PreInitStateCollector),
"Listener should have been initialized with Accord pre-init state collector, but was " + prevListener.getClass());
MetadataChangeListener.PreInitStateCollector preinit = (MetadataChangeListener.PreInitStateCollector) prevListener;
for (ClusterMetadata item : preinit.getItems())
{
if (item.epoch.getEpoch() > highestKnown)
configService.maybeReportMetadata(item);
}
}
catch (InterruptedException e)
{

93
src/java/org/apache/cassandra/service/accord/AccordStaleReplicas.java
View File

@ -20,13 +20,12 @@ package org.apache.cassandra.service.accord;
import java.io.IOException;
import java.util.Objects;
import java.util.Set;
import javax.annotation.concurrent.Immutable;
import com.google.common.collect.ImmutableSet;
import accord.local.Node;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.serializers.TopologySerializers;
@ -36,24 +35,28 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.CollectionSerializers;
import static accord.topology.Topology.NO_IDS;
@Immutable
public class AccordStaleReplicas implements MetadataValue<AccordStaleReplicas>
{
public static final AccordStaleReplicas EMPTY = new AccordStaleReplicas(ImmutableSet.of(), Epoch.EMPTY);
public static final AccordStaleReplicas EMPTY = new AccordStaleReplicas(NO_IDS, NO_IDS, Epoch.EMPTY);
private final Set<Node.Id> staleIds;
private final SortedArrayList<Node.Id> stale;
private final SortedArrayList<Node.Id> hardRemoved;
private final Epoch lastModified;
public AccordStaleReplicas(Set<Node.Id> staleIds, Epoch lastModified)
public AccordStaleReplicas(SortedArrayList<Node.Id> stale, SortedArrayList<Node.Id> hardRemoved, Epoch lastModified)
{
this.staleIds = staleIds;
this.stale = stale;
this.hardRemoved = hardRemoved;
this.lastModified = lastModified;
}
@Override
public AccordStaleReplicas withLastModified(Epoch epoch)
{
return new AccordStaleReplicas(staleIds, epoch);
return new AccordStaleReplicas(stale, hardRemoved, epoch);
}
@Override
@ -62,38 +65,36 @@ public class AccordStaleReplicas implements MetadataValue<AccordStaleReplicas>
return lastModified;
}
public AccordStaleReplicas withNodeIds(Set<Node.Id> ids)
public AccordStaleReplicas withStale(SortedArrayList<Node.Id> stale)
{
ImmutableSet.Builder<Node.Id> builder = new ImmutableSet.Builder<>();
Set<Node.Id> newIds = builder.addAll(staleIds).addAll(ids).build();
return new AccordStaleReplicas(newIds, lastModified);
return new AccordStaleReplicas(this.stale.with(stale), hardRemoved, lastModified);
}
public AccordStaleReplicas without(Set<Node.Id> ids)
public AccordStaleReplicas withHardRemoved(SortedArrayList<Node.Id> hardRemoved)
{
ImmutableSet.Builder<Node.Id> builder = new ImmutableSet.Builder<>();
for (Node.Id staleId : staleIds)
if (!ids.contains(staleId))
builder.add(staleId);
return new AccordStaleReplicas(builder.build(), lastModified);
return new AccordStaleReplicas(stale, this.hardRemoved.with(hardRemoved), lastModified);
}
public boolean contains(Node.Id nodeId)
public AccordStaleReplicas withoutStale(SortedArrayList<Node.Id> withoutStale)
{
return staleIds.contains(nodeId);
return new AccordStaleReplicas(this.stale.without(withoutStale), hardRemoved, lastModified);
}
public Set<Node.Id> ids()
public SortedArrayList<Node.Id> stale()
{
return staleIds;
return stale;
}
public SortedArrayList<Node.Id> hardRemoved()
{
return hardRemoved;
}
@Override
public String toString()
{
return "AccordStaleReplicas{staleIds=" + staleIds + ", lastModified=" + lastModified + '}';
return "AccordStaleReplicas{staleIds=" + stale + ", hardRemoved=" + hardRemoved + ", lastModified=" + lastModified + '}';
}
@Override
@ -102,36 +103,64 @@ public class AccordStaleReplicas implements MetadataValue<AccordStaleReplicas>
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AccordStaleReplicas that = (AccordStaleReplicas) o;
return Objects.equals(staleIds, that.staleIds) && Objects.equals(lastModified, that.lastModified);
return Objects.equals(stale, that.stale) && Objects.equals(lastModified, that.lastModified);
}
@Override
public int hashCode()
{
return Objects.hash(staleIds, lastModified);
return Objects.hash(stale, lastModified);
}
public static final MetadataSerializer<AccordStaleReplicas> serializer = new MetadataSerializer<>()
{
private static final int HAS_STALE_IDS = 0x1;
private static final int HAS_HARD_REMOVED_IDS = 0x2;
@Override
public void serialize(AccordStaleReplicas replicas, DataOutputPlus out, Version version) throws IOException
{
CollectionSerializers.serializeCollection(replicas.staleIds, out, TopologySerializers.nodeId);
int flags = 0;
if (!replicas.stale.isEmpty())
flags |= HAS_STALE_IDS;
if (!replicas.hardRemoved.isEmpty())
flags |= HAS_HARD_REMOVED_IDS;
out.writeUnsignedVInt32(flags);
if (!replicas.stale.isEmpty())
CollectionSerializers.serializeCollection(replicas.stale, out, TopologySerializers.nodeId);
if (!replicas.hardRemoved.isEmpty())
CollectionSerializers.serializeCollection(replicas.hardRemoved, out, TopologySerializers.nodeId);
Epoch.serializer.serialize(replicas.lastModified, out, version);
}
@Override
public AccordStaleReplicas deserialize(DataInputPlus in, Version version) throws IOException
{
return new AccordStaleReplicas(CollectionSerializers.deserializeSet(in, TopologySerializers.nodeId),
Epoch.serializer.deserialize(in, version));
int flags = in.readUnsignedVInt32();
SortedArrayList<Node.Id> stale = NO_IDS, hardRemoved = NO_IDS;
if ((flags & HAS_STALE_IDS) != 0)
stale = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
if ((flags & HAS_HARD_REMOVED_IDS) != 0)
hardRemoved = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
Epoch lastModified = Epoch.serializer.deserialize(in, version);
return new AccordStaleReplicas(stale, hardRemoved, lastModified);
}
@Override
public long serializedSize(AccordStaleReplicas replicas, Version version)
{
return CollectionSerializers.serializedCollectionSize(replicas.staleIds, TopologySerializers.nodeId)
+ Epoch.serializer.serializedSize(replicas.lastModified, version);
int flags = 0;
if (!replicas.stale.isEmpty())
flags |= HAS_STALE_IDS;
if (!replicas.hardRemoved.isEmpty())
flags |= HAS_HARD_REMOVED_IDS;
long size = TypeSizes.sizeofUnsignedVInt(flags);
if (!replicas.stale.isEmpty())
size += CollectionSerializers.serializedCollectionSize(replicas.stale, TopologySerializers.nodeId);
if (!replicas.hardRemoved.isEmpty())
size += CollectionSerializers.serializedCollectionSize(replicas.hardRemoved, TopologySerializers.nodeId);
size += Epoch.serializer.serializedSize(replicas.lastModified, version);
return size;
}
};
}

160
src/java/org/apache/cassandra/service/accord/AccordSyncPropagator.java
View File

@ -22,8 +22,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@ -36,13 +34,14 @@ import accord.local.Node;
import accord.messages.SimpleReply;
import accord.primitives.Ranges;
import accord.utils.Invariants;
import accord.utils.SortedList;
import accord.utils.SortedListSet;
import accord.utils.UnhandledEnum;
import org.agrona.collections.Int2ObjectHashMap;
import org.agrona.collections.Long2ObjectHashMap;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.UnversionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -54,7 +53,6 @@ import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.serializers.TopologySerializers;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.CollectionSerializers;
import org.apache.cassandra.utils.NoSpamLogger;
@ -79,7 +77,7 @@ public class AccordSyncPropagator
interface Listener
{
void onEndpointAck(Node.Id id, long epoch);
void onComplete(long epoch);
void onEndpointNack(Node.Id id, long epoch);
}
private interface ReportPending<T>
@ -191,19 +189,17 @@ public class AccordSyncPropagator
private final Node.Id localId;
private final AccordEndpointMapper endpointMapper;
private final MessageDelivery messagingService;
private final IFailureDetector failureDetector;
private final ScheduledExecutorPlus scheduler;
private final Listener listener;
private final ConcurrentHashMap<RetryKey, Notification> retryingNotifications = new ConcurrentHashMap<>();
public AccordSyncPropagator(Node.Id localId, AccordEndpointMapper endpointMapper,
MessageDelivery messagingService, IFailureDetector failureDetector, ScheduledExecutorPlus scheduler,
MessageDelivery messagingService, ScheduledExecutorPlus scheduler,
Listener listener)
{
this.localId = localId;
this.endpointMapper = endpointMapper;
this.messagingService = messagingService;
this.failureDetector = failureDetector;
this.scheduler = scheduler;
this.listener = listener;
}
@ -234,20 +230,17 @@ public class AccordSyncPropagator
public void onNodesRemoved(Node.Id removed)
{
long[] toAck;
boolean[] syncCompletedFor;
synchronized (AccordSyncPropagator.this)
{
PendingEpochs pendingEpochs = pending.remove(removed.id);
if (pendingEpochs == null) return;
toAck = new long[pendingEpochs.size()];
syncCompletedFor = new boolean[pendingEpochs.size()];
Long2ObjectHashMap<PendingEpoch>.KeyIterator it = pendingEpochs.keySet().iterator();
for (int i = 0; it.hasNext(); i++)
{
long epoch = it.nextLong();
toAck[i] = epoch;
syncCompletedFor[i] = hasSyncCompletedFor(epoch);
}
Arrays.sort(toAck);
}
@ -256,19 +249,15 @@ public class AccordSyncPropagator
{
long epoch = toAck[i];
listener.onEndpointAck(removed, epoch);
if (syncCompletedFor[i])
listener.onComplete(epoch);
}
}
public void reportSyncComplete(long epoch, Collection<Node.Id> notify, Node.Id syncCompleteId)
public void reportCoordinationReady(long epoch, SortedList<Node.Id> notify, Node.Id syncCompleteId)
{
if (notify.isEmpty())
{
listener.onComplete(epoch);
return;
}
report(epoch, notify, PendingEpoch::syncComplete, syncCompleteId);
SortedListSet<Node.Id> remaining = SortedListSet.allOf(notify);
if (remaining.remove(localId))
listener.onEndpointAck(localId, epoch);
report(epoch, remaining, PendingEpoch::syncComplete, syncCompleteId);
}
public void reportClosed(long epoch, Collection<Node.Id> notify, Ranges closed)
@ -281,31 +270,25 @@ public class AccordSyncPropagator
report(epoch, notify, PendingEpoch::retired, retired);
}
private synchronized <T> void report(long epoch, Collection<Node.Id> notify, ReportPending<T> report, T param)
private <T> void report(long epoch, Collection<Node.Id> notify, ReportPending<T> report, T param)
{
// TODO (efficiency, now): for larger clusters this can be a problem as we trigger 1 msg for each instance, so in a 1k cluster its 1k messages; this can cause a thundering herd problem
// this is mostly a problem for reportSyncComplete as we include every node in the cluster, for reportClosed/reportRetired these tend to use only the nodes that are replicas of the range,
// and there is currently an assumption that sub-ranges are done, so only impacting a handful of nodes.
// TODO (correctness, now): during a host replacement multiple epochs are generated (move the range, remove the node), so its possible that notify will never be able to send the notification as the node is leaving the cluster
notify.forEach(id -> {
PendingEpoch pendingEpoch = pending.computeIfAbsent(id.id, ignore -> new PendingEpochs())
.computeIfAbsent(epoch, PendingEpoch::new);
Notification notification = report.report(pendingEpoch, param);
Notification notification;
synchronized (this)
{
PendingEpoch pendingEpoch = pending.computeIfAbsent(id.id, ignore -> new PendingEpochs())
.computeIfAbsent(epoch, PendingEpoch::new);
notification = report.report(pendingEpoch, param);
}
if (notification != null)
notify(id, notification);
});
}
private boolean hasSyncCompletedFor(long epoch)
{
return pending.values().stream().noneMatch(node -> {
PendingEpoch pending = node.get(epoch);
if (pending == null)
return false;
return !pending.syncComplete.isEmpty();
});
}
private void scheduleRetry(Node.Id to, Notification notification)
{
Notification retry = new Notification(notification.epoch, notification.syncComplete, notification.closed, notification.retired, notification.attempts + 1);
@ -329,65 +312,66 @@ public class AccordSyncPropagator
private boolean notify(Node.Id to, Notification notification)
{
InetAddressAndPort toEp = endpointMapper.mappedEndpoint(to);
Message<Notification> msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, notification);
RequestCallback<SimpleReply> cb = new RequestCallback<>()
{
@Override
public void onResponse(Message<SimpleReply> msg)
{
Invariants.require(msg.payload == SimpleReply.Ok, "Unexpected message: %s", msg);
Set<Long> completedEpochs = new HashSet<>();
synchronized (AccordSyncPropagator.this)
{
pending.ack(to, notification);
long epoch = notification.epoch;
if (notification.syncComplete.contains(localId))
{
if (hasSyncCompletedFor(epoch))
completedEpochs.add(epoch);
}
}
long epoch = notification.epoch;
listener.onEndpointAck(to, epoch);
if (completedEpochs.contains(epoch))
listener.onComplete(epoch);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
scheduleRetry(to, notification);
}
@Override
public boolean invokeOnFailure()
{
return true;
}
};
if (!failureDetector.isAlive(toEp))
{
// was the endpoint removed from membership?
ClusterMetadata metadata = ClusterMetadata.current();
if (Gossiper.instance.getEndpointStateForEndpoint(toEp) == null && !metadata.directory.allJoinedEndpoints().contains(toEp) && !metadata.fullCMSMembers().contains(toEp))
{
// endpoint no longer exists...
cb.onResponse(msg.responseWith(SimpleReply.Ok));
return true;
}
noSpamLogger.warn("Node{} is not alive, unable to notify of {}", to, notification);
scheduleRetry(to, notification);
InetAddressAndPort toEp = endpointMapper.mappedEndpointOrNull(to, notification);
if (toEp == null)
return false;
// was the endpoint removed from membership?
AccordEndpointMapper.NodeStatus nodeStatus = endpointMapper.nodeStatus(to);
switch (nodeStatus)
{
default: throw new UnhandledEnum(nodeStatus);
case UNHEALTHY:
if (!endpointMapper.isRemoved(to))
{
noSpamLogger.warn("Node{} is not alive, unable to notify of {}", to, notification);
scheduleRetry(to, notification);
return false;
}
// fall through to UNKNOWN, as we have been removed from the cluster in the latest epoch
case UNKNOWN:
// endpoint is not a member of the latest epoch
pending.ack(to, notification);
listener.onEndpointNack(to, notification.epoch);
return true;
case HEALTHY:
Message<Notification> msg = Message.out(Verb.ACCORD_SYNC_NOTIFY_REQ, notification);
RequestCallback<SimpleReply> cb = new RequestCallback<>()
{
@Override
public void onResponse(Message<SimpleReply> msg)
{
Invariants.require(msg.payload == SimpleReply.Ok, "Unexpected message: %s", msg);
synchronized (AccordSyncPropagator.this)
{
pending.ack(to, notification);
}
long epoch = notification.epoch;
listener.onEndpointAck(to, epoch);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
scheduleRetry(to, notification);
}
@Override
public boolean invokeOnFailure()
{
return true;
}
};
messagingService.sendWithCallback(msg, toEp, cb);
return true;
}
messagingService.sendWithCallback(msg, toEp, cb);
return true;
}
public static class Notification
{
public static final UnversionedSerializer<Notification> serializer = new UnversionedSerializer<Notification>()
public static final UnversionedSerializer<Notification> serializer = new UnversionedSerializer<>()
{
@Override
public void serialize(Notification notification, DataOutputPlus out) throws IOException

46
src/java/org/apache/cassandra/service/accord/AccordTopology.java
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -80,9 +79,9 @@ public class AccordTopology
return new Id(nodeId.id());
}
private static class ShardLookup extends HashMap<accord.primitives.Range, Shard>
public static class ShardLookup extends HashMap<accord.primitives.Range, Shard>
{
private Shard createOrReuse(TinyEnumSet<Shard.Flag> flags, accord.primitives.Range range, SortedArrayList<Id> nodes, SortedArrayList<Id> fastPath, Set<Id> joining)
private Shard createOrReuse(TinyEnumSet<Shard.Flag> flags, accord.primitives.Range range, SortedArrayList<Id> nodes, SortedArrayList<Id> fastPath, SortedArrayList<Id> hardRemoved)
{
Shard prev = get(range);
if (prev != null
@ -90,10 +89,10 @@ public class AccordTopology
&& prev.nodes.equals(nodes)
&& prev.fastPathElectorateSize == fastPath.size()
&& prev.nodes.without(prev.notInFastPath).equals(fastPath)
&& joining.size() == prev.joining.size() && prev.joining.containsAll(joining))
&& hardRemoved.size() == prev.hardRemoved.size() && prev.hardRemoved.containsAll(hardRemoved))
return prev;
return Shard.create(range, nodes, fastPath, joining, flags);
return Shard.create(range, nodes, fastPath, hardRemoved, flags);
}
}
@ -102,14 +101,12 @@ public class AccordTopology
private final KeyspaceMetadata keyspace;
private final List<Range<Token>> ranges;
private final SortedArrayList<Id> nodes;
private final Set<Id> pending;
private KeyspaceShard(KeyspaceMetadata keyspace, List<Range<Token>> ranges, SortedArrayList<Id> nodes, Set<Id> pending)
private KeyspaceShard(KeyspaceMetadata keyspace, List<Range<Token>> ranges, SortedArrayList<Id> nodes)
{
this.keyspace = keyspace;
this.ranges = ranges;
this.nodes = nodes;
this.pending = pending;
}
// return the keyspace fast path strategy if the inherit keyspace strategy is used
@ -122,7 +119,7 @@ public class AccordTopology
return strategy;
}
List<Shard> createForTable(Epoch epoch, TableMetadata metadata, Set<Id> unavailable, Map<Id, String> dcMap, ShardLookup lookup)
List<Shard> createForTable(Epoch epoch, TableMetadata metadata, SortedArrayList<Id> unavailable, SortedArrayList<Id> hardRemoved, Map<Id, String> dcMap, ShardLookup lookup)
{
Ranges ranges = this.ranges.stream()
.map(range -> Ranges.single(AccordTopology.range(metadata.id, range)))
@ -139,7 +136,7 @@ public class AccordTopology
flags = flags.with(Shard.Flag.PENDING_REMOVAL);
if (metadata.epoch.isEqualOrAfter(epoch))
flags = flags.with(Shard.Flag.MUST_WITNESS);
shards.add(lookup.createOrReuse(flags, range, nodes, electorate, pending));
shards.add(lookup.createOrReuse(flags, range, nodes, electorate, hardRemoved.intersecting(nodes)));
}
return shards;
}
@ -159,15 +156,7 @@ public class AccordTopology
.map(AccordTopology::tcmIdToAccord)
.sorted().toArray(Id[]::new));
Set<Id> pending = readEndpoints.equals(writeEndpoints) ?
Collections.emptySet() :
writeEndpoints.stream()
.filter(e -> !readEndpoints.contains(e))
.map(directory::peerId)
.map(AccordTopology::tcmIdToAccord)
.collect(Collectors.toSet());
return new KeyspaceShard(keyspace, ranges, nodes, pending);
return new KeyspaceShard(keyspace, ranges, nodes);
}
public static List<KeyspaceShard> forKeyspace(KeyspaceMetadata keyspace, DataPlacements placements, Directory directory)
@ -217,7 +206,7 @@ public class AccordTopology
return shards;
}
public List<Id> nodes()
public SortedArrayList<Id> nodes()
{
return nodes;
}
@ -296,7 +285,7 @@ public class AccordTopology
AccordStaleReplicas staleReplicas)
{
List<Shard> res = new ArrayList<>();
Set<Id> unavailable = accordFastPath.unavailableIds();
SortedArrayList<Id> unavailable = accordFastPath.unavailableIds();
Map<Id, String> dcMap = createDCMap(directory);
for (KeyspaceMetadata keyspace : schema.getKeyspaces())
@ -305,16 +294,17 @@ public class AccordTopology
if (tables.isEmpty())
continue;
List<KeyspaceShard> ksShards = KeyspaceShard.forKeyspace(keyspace, placements, directory);
tables.forEach(table -> ksShards.forEach(shard -> res.addAll(shard.createForTable(epoch, table, unavailable, dcMap, lookup))));
tables.forEach(table -> ksShards.forEach(shard -> res.addAll(shard.createForTable(epoch, table, unavailable, staleReplicas.hardRemoved(), dcMap, lookup))));
}
res.sort((a, b) -> a.range.compare(b.range));
List<Node.Id> removed = directory.removedNodes().stream()
.filter(n -> n.removedIn.equals(epoch))
.map(n -> tcmIdToAccord(n.id))
.collect(Collectors.toList());
return new Topology(epoch.getEpoch(), SortedArrayList.copySorted(removed, Id[]::new), SortedArrayList.copyUnsorted(staleReplicas.ids(), Id[]::new), res.toArray(new Shard[0]));
SortedArrayList<Node.Id> removed = SortedArrayList.ofUnsorted(
directory.removedNodes().stream()
.filter(n -> n.removedIn.equals(epoch))
.map(n -> tcmIdToAccord(n.id))
.toArray(Id[]::new)
);
return new Topology(epoch.getEpoch(), removed, staleReplicas.hardRemoved(), staleReplicas.stale(), res.toArray(new Shard[0]));
}
public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup)

46
src/java/org/apache/cassandra/service/accord/AccordTracing.java
View File

@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory;
import accord.api.Tracing;
import accord.coordinate.Coordination;
import accord.coordinate.Coordination.CoordinationKind;
import accord.coordinate.tracking.AbstractTracker;
import accord.local.CommandStore;
import accord.local.Node;
import accord.primitives.Participants;
@ -158,9 +157,12 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener
void truncateInternal(int eraseBefore)
{
System.arraycopy(events, eraseBefore, events, 0, size - eraseBefore);
Arrays.fill(events, size - eraseBefore, size, null);
size -= eraseBefore;
if (events != null)
{
System.arraycopy(events, eraseBefore, events, 0, size - eraseBefore);
Arrays.fill(events, size - eraseBefore, size, null);
size -= eraseBefore;
}
}
public TxnEvent get(int index)
@ -314,6 +316,7 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener
if (!globalCount.admit())
return null;
incrementSeen();
return newTrace(kind, null);
}
@ -673,9 +676,11 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener
private synchronized void untrace(TxnId txnId)
{
txnIdMap.compute(txnId, (ignore, cur) -> {
if (cur == null || cur.owner == this)
return null;
return cur;
if (cur == null || cur.owner != this)
return cur;
cur.stopTracing();
return cur.eraseEvents(globalCount);
});
}
@ -968,29 +973,14 @@ public class AccordTracing extends AccordCoordinatorMetrics.Listener
return cur;
event.trace(null, "Failed Coordination Dump");
Coordination.traceStart(event, coordination);
SortedListMap<Node.Id, ?> replies = coordination.replies();
if (replies != null)
{
String description = coordination.describe();
if (description != null)
event.trace(null, "Description: %s", description);
}
{
Participants<?> scope = coordination.scope();
if (scope != null)
event.trace(null, "Scope: %s", scope);
}
{
AbstractTracker<?> tracker = coordination.tracker();
if (tracker != null)
event.trace(null, "Tracker: %s", tracker.summariseTracker());
}
{
SortedListMap<Node.Id, ?> replies = coordination.replies();
if (replies != null)
{
for (int i = 0 ; i < replies.domainSize() ; ++i)
event.trace(null, "from %s: %s", replies.getKey(i), replies.getValue(i));
}
for (int i = 0 ; i < replies.domainSize() ; ++i)
event.trace(null, "from %s: %s", replies.getKey(i), replies.getValue(i));
}
Coordination.traceStop(event, coordination);
return cur;
});
}

8
src/java/org/apache/cassandra/service/accord/AccordVerbHandler.java
View File

@ -60,7 +60,13 @@ public class AccordVerbHandler<T extends Request> implements IVerbHandler<T>
* TODO (desired): messages are retained on heap until the node catches up to waitForEpoch,
* which can be problematic in absense of proper Accord<->Messaging backpressure
*/
Node.Id fromNodeId = endpointMapper.mappedId(message.from());
Node.Id fromNodeId = endpointMapper.mappedIdOrNull(message.from(), message);
if (fromNodeId == null)
{
dropping.debug(message.verb(), message.from());
return;
}
long waitForEpoch = request.waitForEpoch();
if (node.topology().hasAtLeastEpoch(waitForEpoch))
{

107
src/java/org/apache/cassandra/service/accord/EndpointMapping.java
View File

@ -22,17 +22,78 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.utils.Invariants;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.NoSpamLogger;
import static java.util.concurrent.TimeUnit.MINUTES;
class EndpointMapping implements AccordEndpointMapper
{
static class Updateable implements AccordEndpointMapper
{
private volatile EndpointMapping mapping = EMPTY;
@Nullable
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped)
{
return mapping.mappedIdOrNull(endpoint, logIdentityIfUnmapped);
}
@Nullable
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped)
{
return mapping.mappedEndpointOrNull(id, logIdentityIfUnmapped);
}
@Override
public Map<Node.Id, Long> removedNodes()
{
return mapping.removedNodes;
}
@Override
public NodeStatus nodeStatus(Node.Id id)
{
return mapping.nodeStatus(id);
}
@Override
public synchronized void updateMapping(ClusterMetadata metadata)
{
if (metadata.epoch.getEpoch() > this.mapping.epoch())
this.mapping = AccordTopology.directoryToMapping(metadata.epoch.getEpoch(), metadata.directory);
}
public synchronized void updateMapping(EndpointMapping newMapping)
{
if (newMapping.epoch > mapping.epoch)
mapping = newMapping;
}
}
private static final Logger logger = LoggerFactory.getLogger(EndpointMapping.class);
private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, MINUTES);
public static final EndpointMapping EMPTY = new EndpointMapping(0, ImmutableBiMap.of(), ImmutableMap.of());
private final long epoch;
private final ImmutableBiMap<Node.Id, InetAddressAndPort> mapping;
@ -62,21 +123,53 @@ class EndpointMapping implements AccordEndpointMapper
return new ArrayList<>(mapping.keySet());
}
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object logIdentityIfUnmapped)
{
Node.Id id = mapping.inverse().get(endpoint);
if (id != null)
return id;
if (logIdentityIfUnmapped == null) noSpamLogger.warn("Could not find Node.Id for endpoint {}", endpoint);
else noSpamLogger.warn("Could not find Node.Id for endpoint {} on behalf of {}", endpoint, logIdentityIfUnmapped);
return null;
}
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object logIdentityIfUnmapped)
{
InetAddressAndPort ep = mapping.get(id);
if (ep != null)
return ep;
if (logIdentityIfUnmapped == null) noSpamLogger.warn("Could not find InetAddressAndPort for Node.Id {}", id);
else noSpamLogger.warn("Could not find InetAddressAndPort for Node.Id {} on behalf of {}", id, logIdentityIfUnmapped);
return null;
}
@Override
public Map<Node.Id, Long> removedNodes()
{
return removedNodes;
}
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
public NodeStatus nodeStatus(Node.Id id)
{
return mapping.inverse().get(endpoint);
}
InetAddressAndPort ep = mappedEndpointOrNull(id);
if (ep == null)
return NodeStatus.UNKNOWN;
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
{
return mapping.get(id);
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
if (epState == null)
return NodeStatus.UNKNOWN;
if (!epState.isAlive())
return NodeStatus.UNHEALTHY;
VersionedValue event = epState.getApplicationState(ApplicationState.SEVERITY);
if (event == null)
return NodeStatus.HEALTHY; // should we delineate this status better?
return Double.parseDouble(event.value) == 0.0 ? NodeStatus.UNHEALTHY : NodeStatus.HEALTHY;
}
static class Builder

2
src/java/org/apache/cassandra/service/accord/WatermarkCollector.java
View File

@ -81,7 +81,7 @@ public class WatermarkCollector implements ConfigurationService.Listener
synced = new Long2LongHashMap(-1);
}
@Override public AsyncResult<Void> onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync)
@Override public AsyncResult<Void> onTopologyUpdate(Topology topology)
{
return null;
}

47
src/java/org/apache/cassandra/service/accord/api/AccordAgent.java
View File

@ -62,9 +62,11 @@ import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import accord.utils.async.Cancellable;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.exceptions.RequestTimeoutException;
import org.apache.cassandra.metrics.AccordReplicaMetrics;
import org.apache.cassandra.net.ResponseContext;
import org.apache.cassandra.service.RetryStrategy;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTracing;
import org.apache.cassandra.service.accord.serializers.TableMetadatasAndKeys;
@ -88,6 +90,7 @@ import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.fetch
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.recover;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryBootstrap;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryDurability;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retryJoinBootstrap;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.retrySyncPoint;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowTxnPreaccept;
import static org.apache.cassandra.service.accord.api.AccordWaitStrategies.slowRead;
@ -139,10 +142,48 @@ public class AccordAgent implements Agent, OwnershipEventListener
}
@Override
public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure)
public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure)
{
logger.error("Failed bootstrap at {} for {}", phase, ranges, failure);
AccordService.instance().scheduler().once(retry, retryBootstrap.computeWait(attempts, MICROSECONDS), MICROSECONDS);
RetryStrategy strategy;
String message;
SystemKeyspace.BootstrapState bootstrapState = SystemKeyspace.getBootstrapState();
switch (bootstrapState)
{
default: throw new UnhandledEnum(bootstrapState);
case IN_PROGRESS:
case NEEDS_BOOTSTRAP:
message = "Failed bootstrap (for joining) at {} for {}{}";
strategy = retryJoinBootstrap;
break;
case COMPLETED:
case DECOMMISSIONED:
message = "Failed bootstrap at {} for {}{}";
strategy = retryBootstrap;
break;
}
long retryDelayMicros = strategy.computeWait(attempts, MICROSECONDS);
if (retryDelayMicros < 0)
{
if (strategy == retryJoinBootstrap)
{
logger.error(message, phase, ranges, ". Retry strategy giving up. Not yet joined, so failing bootstrap.", failure);
fail.run();
}
else
{
// TODO (expected): we should be able to resume these without restarting (but for now we just shouldn't configure a retry limit)
// failing would prevent the node processing all epochs (as this feeds into the epoch readiness), so we just drop in this case
logger.error(message, phase, ranges, ". Retry strategy giving up. To resume you will need to restart.", failure);
}
}
else
{
logger.error(message, phase, ranges, ". Retrying in " + retryDelayMicros + "us.", failure);
AccordService.instance().scheduler().once(() -> {
logger.info("Retrying bootstrap of {}", ranges);
retry.run();
}, retryDelayMicros, MICROSECONDS);
}
}
@Override

62
src/java/org/apache/cassandra/service/accord/api/AccordTopologySorter.java
View File

@ -22,16 +22,14 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.Set;
import javax.annotation.Nullable;
import accord.api.TopologySorter;
import accord.local.Node;
import accord.topology.ShardSelection;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.utils.SortedList;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.locator.DynamicEndpointSnitch;
import org.apache.cassandra.locator.Endpoint;
import org.apache.cassandra.locator.InetAddressAndPort;
@ -40,6 +38,8 @@ import org.apache.cassandra.service.accord.AccordEndpointMapper;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Sortable;
import static org.apache.cassandra.service.accord.AccordEndpointMapper.NodeStatus.HEALTHY;
public class AccordTopologySorter implements TopologySorter
{
public static class Supplier implements TopologySorter.Supplier
@ -96,54 +96,25 @@ public class AccordTopologySorter implements TopologySorter
@Override
public int compare(Node.Id node1, Node.Id node2, ShardSelection shards)
{
return comparator.compare(() -> mapper.mappedEndpoint(node1), () -> mapper.mappedEndpoint(node2));
InetAddressAndPort ep1 = mapper.mappedEndpointOrNull(node1);
InetAddressAndPort ep2 = mapper.mappedEndpointOrNull(node2);
return compare(comparator, ep1, ep2);
}
@Override
public boolean isFaulty(Node.Id node)
{
InetAddressAndPort ep = mapper.mappedEndpointOrNull(node);
if (ep == null)
return true;
EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(ep);
if (epState == null)
return true;
if (!epState.isAlive())
return true;
VersionedValue event = epState.getApplicationState(ApplicationState.SEVERITY);
if (event == null)
return false;
return Double.parseDouble(event.value) == 0.0;
return mapper.nodeStatus(node) != HEALTHY;
}
private static class EndpointTuple implements Endpoint
{
final InetAddressAndPort endpoint;
private EndpointTuple(InetAddressAndPort endpoint)
{
this.endpoint = endpoint;
}
@Override
public InetAddressAndPort endpoint()
{
return endpoint;
}
}
private static class SortableEndpoints extends ArrayList<EndpointTuple> implements Sortable<EndpointTuple, SortableEndpoints>
private static class SortableEndpoints extends ArrayList<InetAddressAndPort> implements Sortable<InetAddressAndPort, SortableEndpoints>
{
public SortableEndpoints(int initialCapacity)
{
super(initialCapacity);
}
public SortableEndpoints sorted(Comparator<? super EndpointTuple> comparator)
public SortableEndpoints sorted(Comparator<? super InetAddressAndPort> comparator)
{
sort(comparator);
return this;
@ -152,8 +123,19 @@ public class AccordTopologySorter implements TopologySorter
static SortableEndpoints from(Set<Node.Id> nodes, AccordEndpointMapper mapper)
{
SortableEndpoints result = new SortableEndpoints(nodes.size());
nodes.forEach(id -> result.add(new EndpointTuple(mapper.mappedEndpoint(id))));
nodes.forEach(id -> {
InetAddressAndPort ep = mapper.mappedEndpointOrNull(id);
if (ep != null)
result.add(ep);
});
return result;
}
}
private static int compare(Comparator<? super InetAddressAndPort> comparator, @Nullable InetAddressAndPort ep1, @Nullable InetAddressAndPort ep2)
{
if (ep1 == null || ep2 == null)
return ep1 == ep2 ? 0 : ep1 == null ? 1 : -1;
return comparator.compare(ep1, ep2);
}
}

8
src/java/org/apache/cassandra/service/accord/api/AccordWaitStrategies.java
View File

@ -40,7 +40,7 @@ public class AccordWaitStrategies
static TimeoutStrategy slowTxnPreaccept, slowSyncPointPreaccept, slowRead;
static TimeoutStrategy expireTxn, expireSyncPoint, expireDurability, expireEpochWait;
static TimeoutStrategy fetchTxn, fetchSyncPoint;
static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap;
static RetryStrategy recoverTxn, recoverSyncPoint, retrySyncPoint, retryDurability, retryBootstrap, retryJoinBootstrap;
static RetryStrategy retryFetchMinEpoch, retryFetchTopology;
public static @Nullable TimeoutStrategy slowRead(@Nullable TxnId txnId)
@ -98,6 +98,7 @@ public class AccordWaitStrategies
setRetrySyncPoint(config.retry_syncpoint);
setRetryDurability(config.retry_durability);
setRetryBootstrap(config.retry_bootstrap);
setRetryJoinBootstrap(config.retry_join_bootstrap);
setRetryFetchMinEpoch(config.retry_fetch_min_epoch);
setRetryFetchTopology(config.retry_fetch_topology);
}
@ -172,6 +173,11 @@ public class AccordWaitStrategies
retryBootstrap = spec.retry();
}
public static void setRetryJoinBootstrap(StringRetryStrategy spec)
{
retryJoinBootstrap = spec.retry();
}
public static void setRetryFetchMinEpoch(StringRetryStrategy spec)
{
retryFetchMinEpoch = spec.retry();

26
src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java
View File

@ -66,6 +66,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
@ -93,6 +94,7 @@ import org.apache.cassandra.transport.Dispatcher;
import static accord.coordinate.CoordinationAdapter.Factory.Kind.Standard;
import static accord.primitives.Txn.Kind.Write;
import static accord.topology.Topologies.SelectNodeOwnership.SHARE;
import static accord.utils.Invariants.illegalState;
import static accord.utils.Invariants.requireArgument;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordReadMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.accordWriteMetrics;
@ -190,7 +192,10 @@ public class AccordInteropExecution implements ReadCoordinator
for (int i=0; i<replicas.length; i++)
{
Node.Id id = shard.nodes.get(i);
replicas[i] = new Replica(endpointMapper.mappedEndpoint(id), range, true);
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(id);
if (endpoint == null) // TODO (required): how should this be handled?
throw illegalState();
replicas[i] = new Replica(endpoint, range, true);
}
return EndpointsForToken.of(token, replicas);
@ -199,7 +204,12 @@ public class AccordInteropExecution implements ReadCoordinator
@Override
public void sendReadCommand(Message<ReadCommand> message, InetAddressAndPort to, RequestCallback<ReadResponse> callback)
{
Node.Id id = endpointMapper.mappedId(to);
Node.Id id = endpointMapper.mappedIdOrNull(to, message);
if (id == null)
{
callback.onFailure(to, RequestFailure.NODE_DOWN);
return;
}
// TODO (desired): It would be better to use the re-use the command from the transaction but it's fragile
// to try and figure out exactly what changed for things like read repair and short read protection
// Also this read scope doesn't reflect the contents of this particular read and is larger than it needs to be
@ -213,7 +223,12 @@ public class AccordInteropExecution implements ReadCoordinator
{
requireArgument(message.payload.potentialTxnConflicts().allowed);
requireArgument(message.payload.getTableIds().size() == 1);
Node.Id id = endpointMapper.mappedId(to);
Node.Id id = endpointMapper.mappedIdOrNull(to, message);
if (id == null)
{
callback.onFailure(to, RequestFailure.NODE_DOWN);
return;
}
Participants<?> readScope = Participants.singleton(txn.read().keys().domain(), new TokenKey(message.payload.getTableIds().iterator().next(), message.payload.key().getToken()));
AccordInteropReadRepair readRepair = new AccordInteropReadRepair(id, executes, txnId, readScope, executeAt.epoch(), message.payload);
node.send(id, readRepair, executor, new AccordInteropReadRepair.ReadRepairCallback(id, to, message, callback, this));
@ -354,8 +369,11 @@ public class AccordInteropExecution implements ReadCoordinator
private void sendStableToUncontacted()
{
for (Node.Id to : executeTopology.nodes())
if (!contacted.contains(endpointMapper.mappedEndpoint(to)))
{
InetAddressAndPort endpoint = endpointMapper.mappedEndpointOrNull(to);
if (endpoint != null && !contacted.contains(endpoint))
node.send(to, new Commit(Kind.StableFastPath, to, allTopologies, txnId, txn, route, Ballot.ZERO, executeAt, deps));
}
}
public void start()

4
src/java/org/apache/cassandra/service/accord/interop/AccordInteropReadCallback.java
View File

@ -31,7 +31,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.RequestCallback;
import static accord.messages.ReadData.CommitOrReadNack.Insufficient;
import static accord.messages.ReadData.CommitOrReadNack.InsufficientAndWaiting;
public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply>
{
@ -62,7 +62,7 @@ public abstract class AccordInteropReadCallback<T> implements Callback<ReadReply
interopExecution.maybeUpdateUniqueHlc(readOk.uniqueHlc);
wrapped.onResponse(message.responseWith(convertResponse(readOk)).withFrom(endpoint));
}
else if (reply == Insufficient)
else if (reply == InsufficientAndWaiting)
{
// Might still send a response if we send a maximal commit. Accord would tryAlternative and send
// both the commit and an additional repair, but Cassandra doesn't have tryAlternative unless we add

24
src/java/org/apache/cassandra/service/accord/repair/AccordRepair.java
View File

@ -24,6 +24,7 @@ import java.util.List;
import java.util.concurrent.Executor;
import javax.annotation.Nullable;
import accord.local.Node;
import accord.local.durability.DurabilityService.SyncRemote;
import accord.primitives.Ranges;
import accord.primitives.Timestamp;
@ -31,10 +32,12 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.LatencyMetrics;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordEndpointMapper;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTopology;
import org.apache.cassandra.service.accord.IAccordService;
@ -69,18 +72,34 @@ public class AccordRepair
private final Ranges ranges;
private final SyncRemote syncRemote;
private final List<Node.Id> including;
private final Epoch minEpoch = ClusterMetadata.current().epoch;
private volatile Throwable shouldAbort = null;
private volatile Thread waiting;
public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection<Range<Token>> ranges, boolean requireAllEndpoints)
public AccordRepair(SharedContext ctx, ColumnFamilyStore cfs, TimeUUID repairId, String keyspace, Collection<Range<Token>> ranges, boolean requireAllEndpoints, List<InetAddressAndPort> endpoints)
{
this.ctx = ctx;
this.cfs = cfs;
this.repairId = repairId;
// TODO (desired): support unsafe configuration where we permit less than a quorum,
// but this is challenging to do safely as repair has no concept of participants from earlier epochs
this.syncRemote = requireAllEndpoints ? All : Quorum;
if (endpoints != null)
{
including = new ArrayList<>(endpoints.size());
AccordEndpointMapper mapper = AccordService.instance().configService().endpointMapper();
for (InetAddressAndPort ep : endpoints)
{
Node.Id id = mapper.mappedIdOrNull(ep);
if (id == null)
throw new IllegalStateException("Unknown endpoint: " + ep + "; cannot map to Accord Node.Id");
including.add(id);
}
}
else including = null;
this.ranges = AccordTopology.toAccordRanges(keyspace, ranges);
}
@ -141,6 +160,7 @@ public class AccordRepair
private Pair<List<accord.primitives.Range>, Long> repairRange(TokenRange range) throws Throwable
{
List<accord.primitives.Range> repairedRanges = new ArrayList<>();
if (shouldAbort != null)
throw shouldAbort;
@ -164,7 +184,7 @@ public class AccordRepair
long timeoutNanos = getAccordRepairTimeoutNanos();
long maxHlc = AccordService.getBlocking(service.maxConflict(ranges).flatMap(conflict -> {
Timestamp conflictMax = mergeMax(conflict, minForEpoch(this.minEpoch.getEpoch()));
return service.sync("[repairId #" + repairId + ']', conflictMax, Ranges.of(range), null, NoLocal, syncRemote, timeoutNanos, NANOSECONDS).map(ignored -> conflictMax.hlc()).chain();
return service.sync("[repairId #" + repairId + ']', conflictMax, Ranges.of(range), including, NoLocal, syncRemote, timeoutNanos, NANOSECONDS).map(ignored -> conflictMax.hlc()).chain();
}), ranges, bookkeeping, start, start + timeoutNanos);
waiting = null;

20
src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java
View File

@ -62,14 +62,14 @@ public class RecoverySerializers
public void serializeBody(BeginRecovery recover, DataOutputPlus out, Version version) throws IOException
{
CommandSerializers.partialTxn.serialize(recover.partialTxn, out, version);
int flags = (recover.route != null ? HAS_ROUTE : 0)
| (recover.executeAtOrTxnIdEpoch != recover.txnId.epoch() ? HAS_EXECUTE_AT_EPOCH : 0)
| (recover.isFastPathDecided ? IS_FAST_PATH_DECIDED : 0);
int encodedFlags = (recover.route != null ? HAS_ROUTE : 0)
| (recover.executeAtOrTxnIdEpoch != recover.txnId.epoch() ? HAS_EXECUTE_AT_EPOCH : 0)
| recover.flags << 2;
CommandSerializers.ballot.serialize(recover.ballot, out);
out.writeUnsignedVInt32(flags);
out.writeUnsignedVInt32(encodedFlags);
if (recover.route != null)
KeySerializers.fullRoute.serialize(recover.route, out);
if (0 != (flags & HAS_EXECUTE_AT_EPOCH))
if (0 != (encodedFlags & HAS_EXECUTE_AT_EPOCH))
out.writeUnsignedVInt(recover.executeAtOrTxnIdEpoch - recover.txnId.epoch());
}
@ -78,15 +78,15 @@ public class RecoverySerializers
{
PartialTxn partialTxn = CommandSerializers.partialTxn.deserialize(in, version);
Ballot ballot = CommandSerializers.ballot.deserialize(in);
int flags = in.readUnsignedVInt32();
int encodedFlags = in.readUnsignedVInt32();
FullRoute<?> route = null;
if (0 != (flags & HAS_ROUTE))
if (0 != (encodedFlags & HAS_ROUTE))
route = KeySerializers.fullRoute.deserialize(in);
long executeAtOrTxnIdEpoch = txnId.epoch();
if (0 != (flags & HAS_EXECUTE_AT_EPOCH))
if (0 != (encodedFlags & HAS_EXECUTE_AT_EPOCH))
executeAtOrTxnIdEpoch += in.readUnsignedVInt32();
boolean isFastPathDecided = 0 != (flags & IS_FAST_PATH_DECIDED);
return BeginRecovery.SerializationSupport.create(txnId, scope, waitForEpoch, minEpoch, partialTxn, ballot, route, executeAtOrTxnIdEpoch, isFastPathDecided);
int recoveryFlags = encodedFlags >>> 2;
return BeginRecovery.SerializationSupport.create(txnId, scope, waitForEpoch, minEpoch, partialTxn, ballot, route, executeAtOrTxnIdEpoch, recoveryFlags);
}
@Override

10
src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java
View File

@ -19,7 +19,6 @@
package org.apache.cassandra.service.accord.serializers;
import accord.api.Result;
import accord.primitives.ProgressToken;
import org.apache.cassandra.io.UnversionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -27,14 +26,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
public class ResultSerializers
{
// TODO (desired): this is meant to encode e.g. whether the transaction's condition met or not for clients to later query
public static final Result APPLIED = new Result()
{
@Override
public ProgressToken asProgressToken()
{
return ProgressToken.APPLIED;
}
};
public static final Result APPLIED = new Result(){};
public static final UnversionedSerializer<Result> result = new UnversionedSerializer<>()
{

155
src/java/org/apache/cassandra/service/accord/serializers/TopologySerializers.java
View File

@ -41,8 +41,9 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.utils.ArraySerializers;
import org.apache.cassandra.utils.CollectionSerializers;
import org.apache.cassandra.utils.LargeBitSetSerializer;
import org.apache.cassandra.utils.SimpleBitSetSerializers;
import static accord.topology.Topology.NO_IDS;
import static accord.utils.SortedArrays.fromSimpleBitSet;
public class TopologySerializers
@ -112,7 +113,7 @@ public class TopologySerializers
range.serialize(shard.range, out);
CollectionSerializers.serializeList(shard.nodes, out, nodeId);
CollectionSerializers.serializeList(shard.notInFastPath, out, nodeId);
CollectionSerializers.serializeList(shard.joining, out, nodeId);
out.writeUnsignedVInt32(0); // was joining collection, can now be encoding flag bits
out.writeUnsignedVInt32(shard.flags().bitset());
}
@ -122,9 +123,9 @@ public class TopologySerializers
Range range = ShardSerializer.this.range.deserialize(in);
SortedArrayList<Node.Id> nodes = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new);
SortedArrayList<Node.Id> notInFastPath = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new);
SortedArrayList<Node.Id> joining = CollectionSerializers.deserializeSortedArrayList(in, nodeId, Node.Id[]::new);
in.readUnsignedVInt32();
int flags = in.readUnsignedVInt32();
return Shard.SerializerSupport.create(range, nodes, notInFastPath, joining, new TinyEnumSet<>(flags));
return Shard.SerializerSupport.create(range, nodes, notInFastPath, NO_IDS, new TinyEnumSet<>(flags));
}
@Override
@ -133,12 +134,46 @@ public class TopologySerializers
long size = range.serializedSize(shard.range);
size += CollectionSerializers.serializedListSize(shard.nodes, nodeId);
size += CollectionSerializers.serializedListSize(shard.notInFastPath, nodeId);
size += CollectionSerializers.serializedListSize(shard.joining, nodeId);
size += TypeSizes.sizeofUnsignedVInt(0);
size += TypeSizes.sizeofUnsignedVInt(shard.flags().bitset());
return size;
}
}
private static final int HAS_STALE_IDS = 0x1;
private static final int HAS_HARD_REMOVED_IDS = 0x2;
private static void serializeRemovedAndStale(Topology topology, DataOutputPlus out) throws IOException
{
CollectionSerializers.serializeList(topology.removedIds(), out, TopologySerializers.nodeId);
int flags = 0;
if (!topology.staleIds().isEmpty())
flags |= HAS_STALE_IDS;
if (!topology.hardRemovedIds().isEmpty())
flags |= HAS_HARD_REMOVED_IDS;
out.writeUnsignedVInt32(flags);
if (!topology.staleIds().isEmpty())
CollectionSerializers.serializeList(topology.staleIds(), out, TopologySerializers.nodeId);
if (!topology.hardRemovedIds().isEmpty())
CollectionSerializers.serializeList(topology.hardRemovedIds(), out, TopologySerializers.nodeId);
}
private static long serializedSizeOfRemovedAndStale(Topology topology)
{
long size = CollectionSerializers.serializedListSize(topology.removedIds(), TopologySerializers.nodeId);
int flags = 0;
if (!topology.staleIds().isEmpty())
flags |= HAS_STALE_IDS;
if (!topology.hardRemovedIds().isEmpty())
flags |= HAS_HARD_REMOVED_IDS;
size += TypeSizes.sizeofUnsignedVInt(flags);
if (!topology.staleIds().isEmpty())
size += CollectionSerializers.serializedListSize(topology.staleIds(), TopologySerializers.nodeId);
if (!topology.hardRemovedIds().isEmpty())
size += CollectionSerializers.serializedListSize(topology.hardRemovedIds(), TopologySerializers.nodeId);
return size;
}
public static final UnversionedSerializer<Topology> topology = new UnversionedSerializer<>()
{
@Override
@ -146,8 +181,7 @@ public class TopologySerializers
{
out.writeLong(topology.epoch());
CollectionSerializers.serializeList(topology.shards(), out, shard);
CollectionSerializers.serializeCollection(topology.removedIds(), out, TopologySerializers.nodeId);
CollectionSerializers.serializeCollection(topology.staleIds(), out, TopologySerializers.nodeId);
serializeRemovedAndStale(topology, out);
}
@Override
@ -156,8 +190,14 @@ public class TopologySerializers
long epoch = in.readLong();
Shard[] shards = ArraySerializers.deserializeArray(in, shard, Shard[]::new);
SortedArrayList<Node.Id> removedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
SortedArrayList<Node.Id> staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
return new Topology(epoch, removedIds, staleIds, shards);
SortedArrayList<Node.Id> staleIds = NO_IDS, hardRemovedIds = NO_IDS;
int flags = in.readUnsignedVInt32();
if ((flags & HAS_STALE_IDS) != 0)
staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
if ((flags & HAS_HARD_REMOVED_IDS) != 0)
hardRemovedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
// we don't currently serialize hardRemoved at the shard level, so we must re-apply here
return new Topology(epoch, removedIds, NO_IDS, staleIds, shards).withHardRemoved(hardRemovedIds);
}
@Override
@ -166,44 +206,42 @@ public class TopologySerializers
long size = 0;
size += TypeSizes.LONG_SIZE; // epoch
size += CollectionSerializers.serializedListSize(topology.shards(), shard);
size += CollectionSerializers.serializedCollectionSize(topology.removedIds(), TopologySerializers.nodeId);
size += CollectionSerializers.serializedCollectionSize(topology.staleIds(), TopologySerializers.nodeId);
size += serializedSizeOfRemovedAndStale(topology);
return size;
}
};
public static final UnversionedSerializer<Topology> compactTopology = new UnversionedSerializer<>()
{
private final LargeBitSet NO_BITS = new LargeBitSet(0);
private Object2IntHashMap<TokenRange> ranges(Topology topology)
{
// need to loop twice; once to collect ranges, and another to save shards
Object2IntHashMap<TokenRange> result = new Object2IntHashMap<>(-2);
for (Shard shard : topology.shards())
{
TokenRange range = (TokenRange) shard.range;
result.putIfAbsent(range.withTable(TableId.UNDEFINED), -1);
}
int count = 0;
for (Map.Entry<TokenRange, Integer> e : result.entrySet())
e.setValue(count++);
return result;
}
@Override
public void serialize(Topology topology, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt(topology.epoch());
CollectionSerializers.serializeList(topology.removedIds(), out, TopologySerializers.nodeId);
CollectionSerializers.serializeList(topology.staleIds(), out, TopologySerializers.nodeId);
List<Shard> shards = topology.shards();
// need to loop twice; once to collect ranges, and another to save shards
Object2IntHashMap<TokenRange> ranges;
{
Object2IntHashMap<TokenRange> rangesBuilder = new Object2IntHashMap<>(-2);
for (Shard shard : shards)
{
TokenRange range = (TokenRange) shard.range;
rangesBuilder.putIfAbsent(range.withTable(TableId.UNDEFINED), -1);
}
int count = 0;
for (Map.Entry<TokenRange, Integer> e : rangesBuilder.entrySet())
e.setValue(count++);
ranges = rangesBuilder;
}
serializeRemovedAndStale(topology, out);
Object2IntHashMap<TokenRange> ranges = ranges(topology);
CollectionSerializers.serializeCollection(ranges.keySet(), out, TokenRange.noTableSerializer);
out.writeUnsignedVInt32(shards.size());
out.writeUnsignedVInt32(topology.shards().size());
TableId activeTableId = null;
for (Shard shard : shards)
for (Shard shard : topology.shards())
{
TokenRange range = (TokenRange) shard.range;
if (activeTableId == null || !activeTableId.equals(range.table()))
@ -221,9 +259,8 @@ public class TopologySerializers
CollectionSerializers.serializeList(shard.nodes, out, TopologySerializers.nodeId);
LargeBitSet notInFastPath = SortedArrays.toLargeBitSet(shard.nodes, shard.notInFastPath);
LargeBitSetSerializer.instance.serialize(notInFastPath, out);
LargeBitSet joining = SortedArrays.toLargeBitSet(shard.nodes, shard.joining);
LargeBitSetSerializer.instance.serialize(joining, out);
SimpleBitSetSerializers.large.serialize(notInFastPath, out);
SimpleBitSetSerializers.large.serialize(NO_BITS, out);
out.writeUnsignedVInt32(shard.flags().bitset());
}
}
@ -232,32 +269,16 @@ public class TopologySerializers
public long serializedSize(Topology topology)
{
long size = TypeSizes.sizeofUnsignedVInt(topology.epoch());
size += CollectionSerializers.serializedListSize(topology.removedIds(), TopologySerializers.nodeId);
size += CollectionSerializers.serializedListSize(topology.staleIds(), TopologySerializers.nodeId);
List<Shard> shards = topology.shards();
size += serializedSizeOfRemovedAndStale(topology);
// need to loop twice; once to collect ranges, and another to save shards
Object2IntHashMap<TokenRange> ranges;
{
Object2IntHashMap<TokenRange> rangesBuilder = new Object2IntHashMap<>(-2);
for (Shard shard : shards)
{
TokenRange range = (TokenRange) shard.range;
rangesBuilder.putIfAbsent(range.withTable(TableId.UNDEFINED), -1);
}
int count = 0;
for (Map.Entry<TokenRange, Integer> e : rangesBuilder.entrySet())
e.setValue(count++);
ranges = rangesBuilder;
}
Object2IntHashMap<TokenRange> ranges = ranges(topology);
size += CollectionSerializers.serializedCollectionSize(ranges.keySet(), TokenRange.noTableSerializer);
size += TypeSizes.sizeofUnsignedVInt(shards.size());
size += TypeSizes.sizeofUnsignedVInt(topology.shards().size());
TableId activeTableId = null;
for (Shard shard : shards)
for (Shard shard : topology.shards())
{
TokenRange range = (TokenRange) shard.range;
size += TypeSizes.sizeof(true);
@ -272,9 +293,8 @@ public class TopologySerializers
size += CollectionSerializers.serializedListSize(shard.nodes, TopologySerializers.nodeId);
LargeBitSet notInFastPath = SortedArrays.toLargeBitSet(shard.nodes, shard.notInFastPath);
size += LargeBitSetSerializer.instance.serializedSize(notInFastPath);
LargeBitSet joining = SortedArrays.toLargeBitSet(shard.nodes, shard.joining);
size += LargeBitSetSerializer.instance.serializedSize(joining);
size += SimpleBitSetSerializers.large.serializedSize(notInFastPath);
size += SimpleBitSetSerializers.large.serializedSize(NO_BITS);
size += TypeSizes.sizeofUnsignedVInt(shard.flags().bitset());
}
return size;
@ -285,7 +305,14 @@ public class TopologySerializers
{
long epoch = in.readUnsignedVInt();
SortedArrays.SortedArrayList<Node.Id> removedIds = SortedArrays.SortedArrayList.copySorted(CollectionSerializers.deserializeList(in, TopologySerializers.nodeId), Node.Id[]::new);
SortedArrays.SortedArrayList<Node.Id> staleIds = SortedArrays.SortedArrayList.copySorted(CollectionSerializers.deserializeList(in, TopologySerializers.nodeId), Node.Id[]::new);
SortedArrayList<Node.Id> staleIds = NO_IDS, hardRemovedIds = NO_IDS;
{
int flags = in.readUnsignedVInt32();
if ((flags & HAS_STALE_IDS) != 0)
staleIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
if ((flags & HAS_HARD_REMOVED_IDS) != 0)
hardRemovedIds = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
}
List<TokenRange> ranges = CollectionSerializers.deserializeList(in, TokenRange.noTableSerializer);
@ -301,12 +328,12 @@ public class TopologySerializers
TokenRange range = ranges.get(rangeIndex).withTable(activeTableId);
SortedArrays.SortedArrayList<Node.Id> nodes = CollectionSerializers.deserializeSortedArrayList(in, TopologySerializers.nodeId, Node.Id[]::new);
LargeBitSet notInFastPath = LargeBitSetSerializer.instance.deserialize(in);
LargeBitSet joining = LargeBitSetSerializer.instance.deserialize(in);
LargeBitSet notInFastPath = SimpleBitSetSerializers.large.deserialize(in);
SimpleBitSetSerializers.large.deserialize(in);
int flags = in.readUnsignedVInt32();
shards[i] = Shard.SerializerSupport.create(range, nodes, fromSimpleBitSet(nodes, notInFastPath, Node.Id[]::new), fromSimpleBitSet(nodes, joining, Node.Id[]::new), new TinyEnumSet<>(flags));
shards[i] = Shard.SerializerSupport.create(range, nodes, fromSimpleBitSet(nodes, notInFastPath, Node.Id[]::new), NO_IDS, new TinyEnumSet<>(flags));
}
return new Topology(epoch, removedIds, staleIds, shards);
return new Topology(epoch, removedIds, NO_IDS, staleIds, shards).withHardRemoved(hardRemovedIds);
}
};
}

3
src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosBallotTracker.java
View File

@ -197,7 +197,8 @@ public class PaxosBallotTracker
// Accord uses repair (which updates this low bound) to discover the minimum HLC that was used by Paxos
// after Paxos stops. This bound will generally be in the past so it's fine to update it in Accord
// all the time
AccordService.instance().ensureMinHlc(lowBound.unixMicros() + 1);
if (AccordService.isSetup())
AccordService.instance().ensureMinHlc(lowBound.unixMicros() + 1);
}
public Ballot getHighBound()

24
src/java/org/apache/cassandra/tcm/ClusterMetadata.java
View File

@ -21,7 +21,6 @@ package org.apache.cassandra.tcm;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -39,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
@ -462,8 +462,8 @@ public class ClusterMetadata
tokenMap = tokenMap.unassignTokens(nodeId);
Node.Id accordId = AccordTopology.tcmIdToAccord(nodeId);
if (accordStaleReplicas.contains(accordId))
accordStaleReplicas = accordStaleReplicas.without(Collections.singleton(accordId));
if (accordStaleReplicas.stale().contains(accordId))
accordStaleReplicas = accordStaleReplicas.withoutStale(SortedArrayList.ofSorted(accordId));
return this;
}
@ -534,8 +534,8 @@ public class ClusterMetadata
.withNodeState(replacement, NodeState.JOINED);
Node.Id accordId = AccordTopology.tcmIdToAccord(replaced);
if (accordStaleReplicas.contains(accordId))
accordStaleReplicas = accordStaleReplicas.without(Collections.singleton(accordId));
if (accordStaleReplicas.stale().contains(accordId))
accordStaleReplicas = accordStaleReplicas.withoutStale(SortedArrayList.ofSorted(accordId));
return this;
}
@ -566,15 +566,21 @@ public class ClusterMetadata
return this;
}
public Transformer markStaleReplicas(Set<Node.Id> ids)
public Transformer markStaleReplicas(SortedArrayList<Node.Id> markStale)
{
accordStaleReplicas = accordStaleReplicas.withNodeIds(ids);
accordStaleReplicas = accordStaleReplicas.withStale(markStale);
return this;
}
public Transformer unmarkStaleReplicas(Set<Node.Id> ids)
public Transformer markHardRemovedReplicas(SortedArrayList<Node.Id> markHardRemoved)
{
accordStaleReplicas = accordStaleReplicas.without(ids);
accordStaleReplicas = accordStaleReplicas.withHardRemoved(markHardRemoved);
return this;
}
public Transformer unmarkStaleReplicas(SortedArrayList<Node.Id> unmarkStale)
{
accordStaleReplicas = accordStaleReplicas.withoutStale(unmarkStale);
return this;
}

8
src/java/org/apache/cassandra/tcm/Startup.java
View File

@ -447,7 +447,15 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
}
case JOINED:
if (StorageService.isReplacingSameAddress())
{
if (DatabaseDescriptor.getAccordTransactionsEnabled())
{
// TODO (required): we need to support a mode that changes the NodeId when replacing the same address for accord transaction safety
throw new IllegalStateException("Cannot replace same address when accord transactions are enabled.");
}
ReplaceSameAddress.streamData(self, metadata, shouldBootstrap, finishJoiningRing);
}
// JOINED appears before BOOTSTRAPPING & BOOT_REPLACE so we can fall
// through when we start as REGISTERED/LEFT and complete a full startup

2
src/java/org/apache/cassandra/tcm/Transformation.java
View File

@ -38,6 +38,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.AccordMarkHardRemoved;
import org.apache.cassandra.tcm.transformations.AccordMarkRejoining;
import org.apache.cassandra.tcm.transformations.AccordMarkStale;
import org.apache.cassandra.tcm.transformations.AlterSchema;
@ -250,6 +251,7 @@ public interface Transformation
ACCORD_MARK_REJOINING(40, () -> AccordMarkRejoining.serializer),
PREPARE_DROP_ACCORD_TABLE(41, () -> PrepareDropAccordTable.serializer),
FINISH_DROP_ACCORD_TABLE(42, () -> FinishDropAccordTable.serializer),
ACCORD_MARK_HARD_REMOVED(43, () -> AccordMarkHardRemoved.serializer),
;
private final Supplier<AsymmetricMetadataSerializer<Transformation, ? extends Transformation>> serializer;

32
src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java
View File

@ -19,7 +19,9 @@
package org.apache.cassandra.tcm.sequences;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.StreamSupport;
@ -29,6 +31,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.ConfigurationService.EpochReady;
import accord.local.Node;
import accord.utils.async.AsyncResult;
import com.googlecode.concurrenttrees.common.Iterables;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
@ -44,7 +48,7 @@ import org.apache.cassandra.repair.autorepair.AutoRepairUtils;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
@ -362,9 +366,29 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
}
StorageService.instance.repairPaxosForTopologyChange("bootstrap");
Future<StreamState> bootstrapStream = StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements);
Future<?> accordReady = AccordService.instance().epochReadyFor(metadata, EpochReady::reads);
Future<?> ready = FutureCombiner.allOf(bootstrapStream, accordReady);
List<Future<?>> bootstraps = new ArrayList<>();
if (AccordService.instance().isEnabled())
{
IAccordService service = AccordService.instance();
{
Future<?> ready = AccordService.toFuture(service.epochReadyFor(metadata, EpochReady::metadata));
logger.info("Waiting for Accord metadata to be ready");
ready.syncThrowUncheckedOnInterrupt();
ready.rethrowIfFailed();
}
logger.info("Accord metadata is ready, continuing with bootstrap");
bootstraps.add(service.epochReadyFor(metadata, EpochReady::reads));
Node node = service.node();
node.commandStores().forAllUnsafe(commandStore -> {
AsyncResult<EpochReady> ready = commandStore.resumeBootstrap(node);
bootstraps.add(AccordService.toFuture(ready.flatMap(e -> e.reads)));
});
}
bootstraps.add(StorageService.instance.startBootstrap(metadata, beingReplaced, movements, strictMovements));
Future<?> ready = FutureCombiner.allOf(bootstraps);
try
{

152
src/java/org/apache/cassandra/tcm/transformations/AccordMarkHardRemoved.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tcm.transformations;
import java.io.IOException;
import java.util.Objects;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.AccordTopology;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.sequences.BootstrapAndReplace;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.CollectionSerializers;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
public class AccordMarkHardRemoved implements Transformation
{
private static final Logger logger = LoggerFactory.getLogger(AccordMarkHardRemoved.class);
private final Set<NodeId> ids;
private final boolean force;
public AccordMarkHardRemoved(Set<NodeId> ids, boolean force)
{
this.ids = ids;
this.force = force;
}
@Override
public Kind kind()
{
return Kind.ACCORD_MARK_HARD_REMOVED;
}
@Override
public Result execute(ClusterMetadata prev)
{
for (NodeId id : ids)
{
if (!prev.directory.peerIds().contains(id))
continue;
if (!force)
{
boolean removing = false;
for (MultiStepOperation<?> operation : prev.inProgressSequences)
{
switch (operation.kind())
{
case REPLACE:
removing |= id.equals(((BootstrapAndReplace) operation).finishReplace.replaced);
continue;
case REMOVE:
case LEAVE:
removing |= id.equals(((UnbootstrapAndLeave) operation).finishLeave.nodeId);
}
}
if (!removing)
return new Rejected(INVALID, String.format("Cannot mark node %s hard removed as it is still present in the directory.", id));
}
}
SortedArrayList<Node.Id> hardRemoveIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new));
for (Node.Id id : hardRemoveIds)
if (prev.accordStaleReplicas.hardRemoved().contains(id))
return new Rejected(INVALID, String.format("Cannot mark node %s hard removed as it already is.", id));
logger.info("Marking " + ids + " hard removed. These nodes should be permanently offline and unable to respond to any messages at any future point.");
ClusterMetadata.Transformer next = prev.transformer().markHardRemovedReplicas(hardRemoveIds);
return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY);
}
@Override
public String toString()
{
return "AccordMarkStale{ids=" + ids + '}';
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AccordMarkHardRemoved that = (AccordMarkHardRemoved) o;
return Objects.equals(ids, that.ids);
}
@Override
public int hashCode()
{
return Objects.hash(ids);
}
public static final AsymmetricMetadataSerializer<Transformation, AccordMarkHardRemoved> serializer = new AsymmetricMetadataSerializer<>()
{
@Override
public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException
{
assert t instanceof AccordMarkHardRemoved;
AccordMarkHardRemoved mark = (AccordMarkHardRemoved) t;
CollectionSerializers.serializeCollection(mark.ids, out, version, NodeId.serializer);
out.writeBoolean(mark.force);
}
@Override
public AccordMarkHardRemoved deserialize(DataInputPlus in, Version version) throws IOException
{
return new AccordMarkHardRemoved(CollectionSerializers.deserializeSet(in, version, NodeId.serializer), in.readBoolean());
}
@Override
public long serializedSize(Transformation t, Version version)
{
assert t instanceof AccordMarkHardRemoved;
AccordMarkHardRemoved mark = (AccordMarkHardRemoved) t;
return CollectionSerializers.serializedCollectionSize(mark.ids, version, NodeId.serializer) + TypeSizes.BOOL_SIZE;
}
};
}

6
src/java/org/apache/cassandra/tcm/transformations/AccordMarkRejoining.java
View File

@ -21,12 +21,12 @@ package org.apache.cassandra.tcm.transformations;
import java.io.IOException;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.AccordTopology;
@ -64,10 +64,10 @@ public class AccordMarkRejoining implements Transformation
if (!prev.directory.peerIds().contains(id))
return new Rejected(INVALID, String.format("Can not unmark node %s as it is not present in the directory.", id));
Set<Node.Id> accordIds = ids.stream().map(AccordTopology::tcmIdToAccord).collect(Collectors.toSet());
SortedArrayList<Node.Id> accordIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new));
for (Node.Id id : accordIds)
if (!prev.accordStaleReplicas.contains(id))
if (!prev.accordStaleReplicas.stale().contains(id))
return new Rejected(INVALID, String.format("Can not unmark node %s as it is not stale.", id));
logger.info("Unmarking " + ids + ". They will now participate in durability status coordination...");

28
src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java
View File

@ -19,18 +19,16 @@
package org.apache.cassandra.tcm.transformations;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.topology.Shard;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.KeyspaceMetadata;
@ -70,10 +68,11 @@ public class AccordMarkStale implements Transformation
if (!prev.directory.peerIds().contains(id))
return new Rejected(INVALID, String.format("Can not mark node %s stale as it is not present in the directory.", id));
Set<Node.Id> accordIds = ids.stream().map(AccordTopology::tcmIdToAccord).collect(Collectors.toSet());
SortedArrayList<Node.Id> staleIds = SortedArrayList.ofUnsorted(ids.stream().map(AccordTopology::tcmIdToAccord).toArray(Node.Id[]::new));
SortedArrayList<Node.Id> allStaleIds = prev.accordStaleReplicas.stale().with(staleIds);
for (Node.Id id : accordIds)
if (prev.accordStaleReplicas.contains(id))
for (Node.Id id : staleIds)
if (prev.accordStaleReplicas.stale().contains(id))
return new Rejected(INVALID, String.format("Can not mark node %s stale as it already is.", id));
for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES))
@ -82,24 +81,17 @@ public class AccordMarkStale implements Transformation
for (AccordTopology.KeyspaceShard shard : shards)
{
// We're trying to mark a node in this shard stale...
if (!Collections.disjoint(shard.nodes(), accordIds))
SortedArrayList<Node.Id> intersecting = allStaleIds.intersecting(shard.nodes());
if (intersecting.size() > Shard.maxToleratedFailures(shard.nodes().size()))
{
int quorumSize = Shard.slowQuorumSize(shard.nodes().size());
Set<Node.Id> nonStaleNodes = new HashSet<>(shard.nodes());
nonStaleNodes.removeAll(accordIds);
nonStaleNodes.removeAll(prev.accordStaleReplicas.ids());
// ...but reject the transformation if this would bring us below quorum.
if (nonStaleNodes.size() < quorumSize)
return new Rejected(INVALID, String.format("Can not mark nodes %s stale as that would leave fewer than a quorum of nodes active for ranges %s in keyspace '%s'.",
accordIds, shard.ranges(), keyspace.name));
return new Rejected(INVALID, String.format("Can not mark nodes %s stale as that would leave fewer than a quorum of nodes active for ranges %s in keyspace '%s'.",
allStaleIds, shard.ranges(), keyspace.name));
}
}
}
logger.info("Marking " + ids + " stale. They will no longer participate in durability status coordination...");
ClusterMetadata.Transformer next = prev.transformer().markStaleReplicas(accordIds);
ClusterMetadata.Transformer next = prev.transformer().markStaleReplicas(staleIds);
return Transformation.success(next, LockedRanges.AffectedRanges.EMPTY);
}

80
src/java/org/apache/cassandra/utils/LargeBitSetSerializer.java
View File

@ -1,80 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.io.IOException;
import accord.utils.LargeBitSet;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.UnversionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
public class LargeBitSetSerializer implements UnversionedSerializer<LargeBitSet>
{
public static final LargeBitSetSerializer instance = new LargeBitSetSerializer();
@Override
public void serialize(LargeBitSet t, DataOutputPlus out) throws IOException
{
long[] raw = LargeBitSet.SerializationSupport.getArray(t);
// find the first word written
int wordsInUse = wordsInUse(raw);
out.writeUnsignedVInt32(raw.length);
out.writeUnsignedVInt32(wordsInUse);
for (int i = 0; i < wordsInUse; i++)
out.writeUnsignedVInt(raw[i]);
}
@Override
public LargeBitSet deserialize(DataInputPlus in) throws IOException
{
int size = in.readUnsignedVInt32();
long[] raw = new long[size];
int wordsInUse = in.readUnsignedVInt32();
for (int i = 0; i < wordsInUse; i++)
raw[i] = in.readUnsignedVInt();
return LargeBitSet.SerializationSupport.construct(raw);
}
@Override
public long serializedSize(LargeBitSet t)
{
long[] raw = LargeBitSet.SerializationSupport.getArray(t);
// find the last word written
int wordsInUse = wordsInUse(raw);
long size = TypeSizes.sizeofUnsignedVInt(raw.length);
size += TypeSizes.sizeofVInt(wordsInUse);
for (int i = 0; i < wordsInUse; i++)
size += TypeSizes.sizeofUnsignedVInt(raw[i]);
return size;
}
private static int wordsInUse(long[] raw)
{
int wordsInUse = raw.length;
for (int i = raw.length - 1; i >= 0; i--)
{
if (raw[i] != 0)
return wordsInUse;
wordsInUse--;
}
return wordsInUse;
}
}

2
src/java/org/apache/cassandra/utils/MerkleTree.java
View File

@ -735,7 +735,7 @@ public class MerkleTree
if (offHeapRequested && !offHeapSupported && !warnedOnce)
{
logger.warn("Configuration requests off-heap merkle trees, but partitioner does not support it. Ignoring.");
logger.warn("Configuration requests off-heap merkle trees, but partitioner {} does not support it. Ignoring.", partitioner.getClass().getName());
warnedOnce = true;
}

193
src/java/org/apache/cassandra/utils/SimpleBitSetSerializers.java Normal file
View File

@ -0,0 +1,193 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.io.IOException;
import accord.utils.Invariants;
import accord.utils.LargeBitSet;
import accord.utils.SimpleBitSet;
import accord.utils.SmallBitSet;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.UnversionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
public class SimpleBitSetSerializers
{
public static final AnySerializer any = new AnySerializer();
public static final SmallSerializer small = new SmallSerializer();
public static final SmallWithCapacitySerializer smallWithCapacity = new SmallWithCapacitySerializer();
public static final LargeSerializer large = new LargeSerializer();
public static class SmallSerializer implements UnversionedSerializer<SmallBitSet>
{
@Override
public void serialize(SmallBitSet t, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt(t.bits());
}
@Override
public SmallBitSet deserialize(DataInputPlus in) throws IOException
{
return new SmallBitSet(in.readUnsignedVInt());
}
@Override
public long serializedSize(SmallBitSet t)
{
return TypeSizes.sizeofUnsignedVInt(t.bits());
}
}
public static class SmallWithCapacitySerializer extends SmallSerializer
{
@Override
public void serialize(SmallBitSet t, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt32(1);
super.serialize(t, out);
}
@Override
public SmallBitSet deserialize(DataInputPlus in) throws IOException
{
Invariants.require(in.readUnsignedVInt32() <= 1);
return super.deserialize(in);
}
@Override
public long serializedSize(SmallBitSet t)
{
return TypeSizes.sizeofUnsignedVInt(1) + super.serializedSize(t);
}
}
public static class LargeSerializer implements UnversionedSerializer<LargeBitSet>
{
@Override
public void serialize(LargeBitSet t, DataOutputPlus out) throws IOException
{
long[] raw = LargeBitSet.SerializationSupport.getArray(t);
out.writeUnsignedVInt32(raw.length);
if (raw.length <= 1)
{
out.writeUnsignedVInt(raw.length == 0 ? 0L : raw[0]);
}
else
{
// find the first word written
int wordsInUse = wordsInUse(raw);
out.writeUnsignedVInt32(wordsInUse);
for (int i = 0; i < wordsInUse; i++)
out.writeUnsignedVInt(raw[i]);
}
}
@Override
public LargeBitSet deserialize(DataInputPlus in) throws IOException
{
int capacity = in.readUnsignedVInt32();
return deserializeWithCapacity(capacity, in);
}
LargeBitSet deserializeWithCapacity(int capacityInLongs, DataInputPlus in) throws IOException
{
if (capacityInLongs <= 1)
{
long v = in.readUnsignedVInt();
Invariants.require(capacityInLongs == 1 || v == 0);
long[] raw = capacityInLongs == 0 ? new long[0] : new long[] { v };
return LargeBitSet.SerializationSupport.construct(raw);
}
long[] raw = new long[capacityInLongs];
int wordsInUse = in.readUnsignedVInt32();
for (int i = 0; i < wordsInUse; i++)
raw[i] = in.readUnsignedVInt();
return LargeBitSet.SerializationSupport.construct(raw);
}
@Override
public long serializedSize(LargeBitSet t)
{
long[] raw = LargeBitSet.SerializationSupport.getArray(t);
return TypeSizes.sizeofUnsignedVInt(raw.length)
+ serializedSizeWithoutCapacity(raw);
}
private long serializedSizeWithoutCapacity(long[] raw)
{
if (raw.length <= 1)
return TypeSizes.sizeofUnsignedVInt(raw.length == 0 ? 0L : raw[0]);
// find the last word written
int wordsInUse = wordsInUse(raw);
long size = TypeSizes.sizeofUnsignedVInt(wordsInUse);
for (int i = 0; i < wordsInUse; i++)
size += TypeSizes.sizeofUnsignedVInt(raw[i]);
return size;
}
private static int wordsInUse(long[] raw)
{
int wordsInUse = raw.length;
for (int i = raw.length - 1; i >= 0; i--)
{
if (raw[i] != 0)
return wordsInUse;
wordsInUse--;
}
return wordsInUse;
}
}
private static <S extends SimpleBitSet> UnversionedSerializer<S> serializer(SimpleBitSet set)
{
if (set.getClass() == LargeBitSet.class) return (UnversionedSerializer<S>) large;
if (set.getClass() == SmallBitSet.class) return (UnversionedSerializer<S>) smallWithCapacity;
throw new UnsupportedOperationException("Unknown bitset type: " + set.getClass());
}
public static class AnySerializer implements UnversionedSerializer<SimpleBitSet>
{
@Override
public void serialize(SimpleBitSet bitset, DataOutputPlus out) throws IOException
{
serializer(bitset).serialize(bitset, out);
}
@Override
public SimpleBitSet deserialize(DataInputPlus in) throws IOException
{
int capacity = in.readUnsignedVInt32();
// can use small or large deserializer for capacity <= 1, but to ensure capacity() is the same we use small serializer only when capacity == 1
if (capacity == 1) return small.deserialize(in);
else return large.deserializeWithCapacity(capacity, in);
}
@Override
public long serializedSize(SimpleBitSet bitset)
{
return serializer(bitset).serializedSize(bitset);
}
}
}

23
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -337,11 +337,26 @@ public class ClusterUtils
I toReplace,
BiConsumer<I, WithProperties> fn)
{
IInstanceConfig toReplaceConf = toReplace.config();
I inst = addInstance(cluster, toReplaceConf, c -> c.set("auto_bootstrap", true)
.set("progress_barrier_min_consistency_level", ConsistencyLevel.ONE));
return startHostReplacement(toReplace, inst, fn);
return replaceHostAndStart(cluster, toReplace, fn, ignore -> {});
}
public static <I extends IInstance> I replaceHostAndStart(AbstractCluster<I> cluster,
I toReplace,
BiConsumer<I, WithProperties> fn,
Consumer<IInstanceConfig> configFn)
{
IInstanceConfig toReplaceConf = toReplace.config();
I inst = addInstance(cluster, toReplaceConf, c -> {
c.set("auto_bootstrap", true)
.set("progress_barrier_min_consistency_level", ConsistencyLevel.ONE);
configFn.accept(c);
});
return startHostReplacement(toReplace, inst, fn);
}
public static <I extends IInstance> I startHostReplacement(I toReplace, I inst)
{
return startHostReplacement(toReplace, inst, (i1, i2) -> {});
}
/**

281
test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTest.java
View File

@ -24,77 +24,55 @@ import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import org.junit.Assert;
import org.junit.Test;
import accord.api.ConfigurationService.EpochReady;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.topology.TopologyManager;
import accord.utils.async.AsyncResult;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordConfigurationService;
import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.assertj.core.api.Assertions;
import org.apache.cassandra.streaming.StreamState;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.apache.cassandra.Util.spinUntilTrue;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.service.accord.AccordService.getBlocking;
import static org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot.ResultStatus.SUCCESS;
import static org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus.COMPLETED;
public class AccordBootstrapTest extends TestBaseImpl
public class AccordBootstrapTest extends AccordBootstrapTestBase
{
private static DecoratedKey dk(int key)
{
IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
return partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
private static PartitionKey pk(int key, String keyspace, String table)
{
TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
return new PartitionKey(tid, dk(key));
}
protected IInvokableInstance bootstrapAndJoinNode(Cluster cluster)
protected IInvokableInstance failedAndResumeBootstrapAndJoinNode(Cluster cluster)
{
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
config.set("accord.shard_durability_target_splits", "1");
config.set("accord.shard_durability_cycle", "20s");
config.set("accord.retry_join_bootstrap", "10s,attempts=1");
cluster.forEach(instance -> instance.runOnInstance(() -> StreamListener.listener.failStream = true));
IInvokableInstance newInstance = cluster.bootstrap(config);
newInstance.startup(cluster);
spinUntilTrue(() -> cluster.stream().anyMatch(instance -> instance.callOnInstance(() -> StreamListener.listener.hasFailedStream)));
newInstance.shutdown(false);
cluster.get(1, 2).forEach(instance -> instance.runOnInstance(() -> StreamListener.listener.failStream = false));
newInstance.startup(cluster);
// todo: re-add once we fix write survey/join ring = false mode
// withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000),
// () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster)));
@ -103,54 +81,13 @@ public class AccordBootstrapTest extends TestBaseImpl
return newInstance;
}
private static AccordService service()
{
return (AccordService) AccordService.instance();
}
private static void awaitEpoch(long epoch, Function<EpochReady, AsyncResult<Void>> await)
{
try
{
boolean completed = service().epochReady(Epoch.create(epoch), await).await(60, TimeUnit.SECONDS);
Assertions.assertThat(completed)
.describedAs("Epoch %s did not become ready within timeout on %s -> %s",
epoch, FBUtilities.getBroadcastAddressAndPort(),
service().configService().getEpochSnapshot(epoch))
.isTrue();
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
private static void awaitLocalSyncNotification(long epoch)
{
try
{
AccordConfigurationService configService = service().configService();
boolean completed = configService.unsafeLocalSyncNotified(epoch).await(30, TimeUnit.SECONDS);
Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s\n%s",
epoch, FBUtilities.getBroadcastAddressAndPort(), service().configService().getDebugStr()), completed);
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
private static long maxEpoch(Cluster cluster)
{
return cluster.stream().mapToLong(node -> node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch())).max().getAsLong();
}
private static class StreamListener implements StreamManager.StreamListener
{
private static boolean isRegistered = false;
private static final StreamListener listener = new StreamListener();
private final List<StreamResultFuture> registered = new ArrayList<>();
private boolean failStream, hasFailedStream;
static synchronized void register()
{
@ -163,6 +100,34 @@ public class AccordBootstrapTest extends TestBaseImpl
public synchronized void onRegister(StreamResultFuture result)
{
registered.add(result);
if (failStream)
{
result.addEventListener(new StreamEventHandler()
{
@Override
public void handleStreamEvent(StreamEvent event)
{
if (event.eventType == StreamEvent.Type.STREAM_PREPARED)
{
result.getCoordinator().getAllStreamSessions().forEach(StreamSession::abort);
hasFailedStream = true;
}
}
@Override
public void onSuccess(StreamState result)
{
}
@Override
public void onFailure(Throwable t)
{
}
}
);
}
}
public synchronized void forSession(Consumer<StreamSession> consumer)
@ -176,13 +141,22 @@ public class AccordBootstrapTest extends TestBaseImpl
@Test
public void bootstrapTest() throws Throwable
{
bootstrapTest(Function.identity(), cluster -> {
bootstrapTest(cluster -> {
bootstrapAndJoinNode(cluster);
awaitMaxEpochReadyToRead(cluster);
});
}
public void bootstrapTest(Function<Cluster.Builder, Cluster.Builder> setup, Consumer<Cluster> bootstrapAndJoinNode) throws Throwable
@Test
public void resumeBootstrapTest() throws Throwable
{
bootstrapTest(cluster -> {
failedAndResumeBootstrapAndJoinNode(cluster);
awaitMaxEpochReadyToRead(cluster);
});
}
public void bootstrapTest(Consumer<Cluster> bootstrapAndJoinNode) throws Throwable
{
int originalNodeCount = 2;
int expandedNodeCount = originalNodeCount + 1;
@ -286,18 +260,18 @@ public class AccordBootstrapTest extends TestBaseImpl
Assert.assertFalse(ss.bootstrapBeganAt().isEmpty());
Assert.assertFalse(ss.safeToReadAt().isEmpty());
Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
Assert.assertTrue(ss.bootstrapBeganAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.anyMatch(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return true;
}));
Assert.assertTrue(ss.safeToReadAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.anyMatch(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return true;
}));
}
}));
}
@ -309,131 +283,4 @@ public class AccordBootstrapTest extends TestBaseImpl
});
}
}
@Test
public void moveTest() throws Throwable
{
try (Cluster cluster = Cluster.build().withNodes(3)
.withoutVNodes()
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0"))
.withConfig(config -> config
.set("accord.shard_durability_target_splits", "1")
.set("accord.shard_durability_cycle", "20s")
.with(NETWORK, GOSSIP))
.start())
{
cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}");
cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'");
long initialMax = maxEpoch(cluster);
long[] tokens = new long[3];
for (int i=0; i<3; i++)
{
tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens())));
}
awaitMaxEpochReadyToRead(cluster);
for (int key = 0; key < 100; key++)
{
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
" END IF\n" +
"COMMIT TRANSACTION";
AccordTestBase.executeWithRetry(cluster, query);
}
long token = ((tokens[1] - tokens[0]) / 2) + tokens[0];
long preMove = maxEpoch(cluster);
cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token)));
long moveMax = awaitMaxEpochReadyToRead(cluster);
for (IInvokableInstance node : cluster)
{
node.runOnInstance(() -> {
// validate streaming
List<Range<Token>> ranges = StorageService.instance.getLocalRanges("ks");
TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id;
for (int key = 0; key < 100; key++)
{
DecoratedKey dk = dk(key);
UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key);
if (ranges.stream().anyMatch(range -> range.contains(dk.getToken())))
{
UntypedResultSet.Row row = getOnlyElement(result);
Assert.assertEquals(key, row.getInt("c"));
Assert.assertEquals(key, row.getInt("v"));
PartitionKey partitionKey = new PartitionKey(tableId, dk);
getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> {
if (!safeStore.ranges().allAt(preMove).contains(partitionKey))
{
AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore;
Assert.assertFalse(ss.bootstrapBeganAt().isEmpty());
Assert.assertFalse(ss.safeToReadAt().isEmpty());
Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
}
}));
}
}
});
}
}
}
private static long awaitMaxEpochReadyToRead(Cluster cluster)
{
return awaitMaxEpoch(cluster, EpochReady::reads, true);
}
private static long awaitMaxEpochMetadataReady(Cluster cluster)
{
return awaitMaxEpoch(cluster, EpochReady::metadata, false);
}
private static long awaitMaxEpoch(Cluster cluster, SerializableFunction<EpochReady, AsyncResult<Void>> await, boolean expectReadyToRead)
{
long maxEpoch = maxEpoch(cluster);
for (IInvokableInstance node : cluster)
{
node.acceptOnInstance(aw -> {
ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(maxEpoch));
Assert.assertEquals(maxEpoch, ClusterMetadata.current().epoch.getEpoch());
AccordService service = (AccordService) AccordService.instance();
awaitEpoch(maxEpoch, aw);
AccordConfigurationService configService = service.configService();
awaitLocalSyncNotification(maxEpoch);
for (long epoch = configService.minEpoch(); epoch <= maxEpoch; epoch++)
{
Assert.assertEquals(COMPLETED, configService.getEpochSnapshot(maxEpoch).syncStatus);
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).acknowledged);
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).received);
if (expectReadyToRead)
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).reads);
}
}, node.transfer(await));
}
return maxEpoch;
}
}

165
test/distributed/org/apache/cassandra/distributed/test/accord/AccordBootstrapTestBase.java Normal file
View File

@ -0,0 +1,165 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.junit.Assert;
import accord.api.ConfigurationService.EpochReady;
import accord.utils.async.AsyncResult;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordConfigurationService;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.service.accord.AccordConfigurationService.EpochSnapshot.ResultStatus.SUCCESS;
import static org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus.COMPLETED;
public class AccordBootstrapTestBase extends TestBaseImpl
{
static DecoratedKey dk(int key)
{
IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
return partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
static PartitionKey pk(int key, String keyspace, String table)
{
TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
return new PartitionKey(tid, dk(key));
}
protected IInvokableInstance bootstrapAndJoinNode(Cluster cluster)
{
IInstanceConfig config = cluster.newInstanceConfig();
config.set("auto_bootstrap", true);
config.set("accord.shard_durability_target_splits", "1");
config.set("accord.shard_durability_cycle", "20s");
IInvokableInstance newInstance = cluster.bootstrap(config);
newInstance.startup(cluster);
// todo: re-add once we fix write survey/join ring = false mode
// withProperty(BOOTSTRAP_SCHEMA_DELAY_MS.getKey(), Integer.toString(90 * 1000),
// () -> withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster)));
// newInstance.nodetoolResult("join").asserts().success();
newInstance.nodetoolResult("cms", "describe").asserts().success(); // just make sure we're joined, remove later
return newInstance;
}
static AccordService service()
{
return (AccordService) AccordService.instance();
}
static void awaitEpoch(long epoch, Function<EpochReady, AsyncResult<Void>> await)
{
try
{
boolean completed = service().epochReady(Epoch.create(epoch), await).await(60, TimeUnit.SECONDS);
Assertions.assertThat(completed)
.describedAs("Epoch %s did not become ready within timeout on %s -> %s",
epoch, FBUtilities.getBroadcastAddressAndPort(),
service().configService().getEpochSnapshot(epoch))
.isTrue();
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
static void awaitLocalSyncNotification(long epoch)
{
try
{
AccordConfigurationService configService = service().configService();
boolean completed = configService.unsafeLocalSyncNotified(epoch).await(60, TimeUnit.SECONDS);
Assert.assertTrue(String.format("Local sync notification for epoch %s did not become ready within timeout on %s\n%s",
epoch, FBUtilities.getBroadcastAddressAndPort(), service().configService().getDebugStr()), completed);
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
static long maxEpoch(Cluster cluster)
{
return cluster.stream().mapToLong(node -> {
if (node.isShutdown())
return 0L;
return node.callOnInstance(() -> ClusterMetadata.current().epoch.getEpoch());
}).max().getAsLong();
}
static long awaitMaxEpochReadyToRead(Cluster cluster)
{
return awaitMaxEpoch(cluster, EpochReady::reads, true);
}
static long awaitMaxEpochMetadataReady(Cluster cluster)
{
return awaitMaxEpoch(cluster, EpochReady::metadata, false);
}
static long awaitMaxEpoch(Cluster cluster, SerializableFunction<EpochReady, AsyncResult<Void>> await, boolean expectReadyToRead)
{
long maxEpoch = maxEpoch(cluster);
for (IInvokableInstance node : cluster)
{
if (node.isShutdown())
continue;
node.acceptOnInstance(aw -> {
ClusterMetadataService.instance().fetchLogFromCMS(Epoch.create(maxEpoch));
Assert.assertEquals(maxEpoch, ClusterMetadata.current().epoch.getEpoch());
AccordService service = (AccordService) AccordService.instance();
awaitEpoch(maxEpoch, aw);
AccordConfigurationService configService = service.configService();
awaitLocalSyncNotification(maxEpoch);
for (long epoch = configService.minEpoch(); epoch <= maxEpoch; epoch++)
{
Assert.assertEquals(COMPLETED, configService.getEpochSnapshot(maxEpoch).syncStatus);
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).acknowledged);
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).received);
if (expectReadyToRead)
Assert.assertEquals(SUCCESS, configService.getEpochSnapshot(maxEpoch).reads);
}
}, node.transfer(await));
}
return maxEpoch;
}
}

145
test/distributed/org/apache/cassandra/distributed/test/accord/AccordCommandStoreTryExecuteListeningTest.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import org.junit.Test;
import accord.api.Journal;
import accord.local.Command;
import accord.local.Node;
import accord.local.PreLoadContext;
import accord.local.PreLoadContext.Empty;
import accord.local.StoreParticipants;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.KeyDeps;
import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.RoutingKeys;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.utils.ImmutableBitSet;
import accord.utils.LargeBitSet;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
public class AccordCommandStoreTryExecuteListeningTest extends TestBaseImpl
{
private static DecoratedKey dk(int key)
{
IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
return partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
private static PartitionKey pk(int key, String keyspace, String table)
{
TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
return new PartitionKey(tid, dk(key));
}
@Test
public void testTryExecuteListening() throws Throwable
{
try (Cluster cluster = Cluster.build().withNodes(1)
.withoutVNodes()
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(1, "dc0", "rack0"))
.withConfig(config -> config.set("accord.command_store_shard_count", 1)
.set("accord.queue_shard_count", 1)
.with(NETWORK, GOSSIP))
.start())
{
cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':1}");
cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'");
cluster.get(1).runOnInstance(() -> {
AccordService service = (AccordService) AccordService.instance();
Node node = service.node();
PartitionKey key = pk(1, "ks", "tbl");
AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable());
Command txn1a = executed(node, SaveStatus.Applied);
Command txn1b = executed(node, SaveStatus.PreApplied);
Command txn2a = executed(node, SaveStatus.PreApplied, txn1a.txnId());
Command txn2b = executed(node, SaveStatus.PreApplied, txn1b.txnId());
Command txn3 = executed(node, SaveStatus.PreApplied, txn1a.txnId(), txn1b.txnId(), txn2b.txnId());
Command txn4 = executed(node, SaveStatus.PreApplied, txn1a.txnId(), txn1b.txnId(), txn3.txnId());
Command[] commands = new Command[] { txn1a, txn1b, txn2a, txn2b, txn3, txn4 };
AccordService.getBlocking(commandStore.chain((Empty)() -> "Test", safeStore -> {
for (Command command : commands)
commandStore.journal.saveCommand(commandStore.id(), new Journal.CommandUpdate(null, command), () -> {});
commandStore.unsafeGetListeners().register(txn1a.txnId(), SaveStatus.Applied, txn2a.txnId());
commandStore.unsafeGetListeners().register(txn3.txnId(), SaveStatus.Applied, txn4.txnId());
}));
AccordService.getBlocking(commandStore.operatorTryToExecuteListeningTxns());
for (Command command : commands)
{
Command cmd = AccordService.getBlocking(commandStore.submit(PreLoadContext.contextFor(command.txnId(), "Test"), safeStore -> safeStore.unsafeGet(command.txnId()).current()));
Assertions.assertThat(cmd.saveStatus()).isEqualTo(SaveStatus.Applied);
}
});
}
}
private static Command executed(Node node, SaveStatus saveStatus, TxnId ... dependencies)
{
PartitionKey key = pk(1, "ks", "tbl");
Txn txn = node.agent().emptySystemTxn(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range);
TxnId txnId = node.nextTxnId(txn);
FullRoute<?> route = node.computeRoute(txnId, Ranges.of(key.asRange()));
AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable());
int[] rangesToTxnIds = new int[dependencies.length + 1];
rangesToTxnIds[0] = rangesToTxnIds.length;
for (int i = 1; i < rangesToTxnIds.length ; ++i)
rangesToTxnIds[i] = i - 1;
Deps deps = new Deps(KeyDeps.NONE, RangeDeps.SerializerSupport.create(new accord.primitives.Range[] { key.asRange() }, dependencies, rangesToTxnIds, null));
Command.WaitingOn waitingOn; {
LargeBitSet waitingOnBits = new LargeBitSet(dependencies.length);
waitingOnBits.setRange(0, dependencies.length);
waitingOn = new Command.WaitingOn(RoutingKeys.EMPTY, deps.rangeDeps, new ImmutableBitSet(waitingOnBits), new ImmutableBitSet(dependencies.length));
}
return Command.Executed.executed(txnId, saveStatus, Status.Durability.NotDurable, StoreParticipants.execute(commandStore.unsafeGetRangesForEpoch(), route, txnId, txnId.epoch()), Ballot.ZERO, txnId, txn.intersecting(route, true), deps.intersecting(route), Ballot.ZERO, waitingOn, null, ResultSerializers.APPLIED);
}
}

139
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMoveTest.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.api.PartitionKey;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.service.accord.AccordService.getBlocking;
public class AccordMoveTest extends AccordBootstrapTestBase
{
@Test
public void moveTest() throws Throwable
{
try (Cluster cluster = Cluster.build().withNodes(3)
.withoutVNodes()
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(3))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(3, "dc0", "rack0"))
.withConfig(config -> config
.set("accord.shard_durability_target_splits", "1")
.set("accord.shard_durability_cycle", "20s")
.with(NETWORK, GOSSIP))
.start())
{
cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}");
cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'");
long initialMax = maxEpoch(cluster);
long[] tokens = new long[3];
for (int i=0; i<3; i++)
{
tokens[i] = cluster.get(i+1).callOnInstance(() -> Long.valueOf(getOnlyElement(StorageService.instance.getTokens())));
}
awaitMaxEpochReadyToRead(cluster);
for (int key = 0; key < 100; key++)
{
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
" END IF\n" +
"COMMIT TRANSACTION";
AccordTestBase.executeWithRetry(cluster, query);
}
long token = ((tokens[1] - tokens[0]) / 2) + tokens[0];
long preMove = maxEpoch(cluster);
cluster.get(1).runOnInstance(() -> StorageService.instance.move(Long.toString(token)));
long moveMax = awaitMaxEpochReadyToRead(cluster);
for (IInvokableInstance node : cluster)
{
node.runOnInstance(() -> {
// validate streaming
List<Range<Token>> ranges = StorageService.instance.getLocalRanges("ks");
TableId tableId = Schema.instance.getTableMetadata("ks", "tbl").id;
for (int key = 0; key < 100; key++)
{
DecoratedKey dk = dk(key);
UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key);
if (ranges.stream().anyMatch(range -> range.contains(dk.getToken())))
{
UntypedResultSet.Row row = getOnlyElement(result);
Assert.assertEquals(key, row.getInt("c"));
Assert.assertEquals(key, row.getInt("v"));
PartitionKey partitionKey = new PartitionKey(tableId, dk);
getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), moveMax, moveMax, safeStore -> {
if (!safeStore.ranges().allAt(preMove).contains(partitionKey))
{
AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore;
Assert.assertFalse(ss.bootstrapBeganAt().isEmpty());
Assert.assertFalse(ss.safeToReadAt().isEmpty());
Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
}
}));
}
}
});
}
}
}
}

10
test/distributed/org/apache/cassandra/distributed/test/accord/AccordNodetoolTest.java
View File

@ -43,7 +43,7 @@ public class AccordNodetoolTest extends TestBaseImpl
try (Cluster cluster = init(builder().withNodes(3).withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)).start()))
{
cluster.get(1).nodetoolResult("accord", "mark_stale", "1").asserts().success();
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids()));
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale()));
cluster.get(1).nodetoolResult("accord", "describe").asserts().stdoutContains("Stale Replicas: 1");
// Reject the operation if the target node is already stale:
@ -56,7 +56,7 @@ public class AccordNodetoolTest extends TestBaseImpl
cluster.get(1).nodetoolResult("accord", "mark_stale", "4").asserts().failure().errorContains("not present in the directory");
cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1").asserts().success();
cluster.get(1).runOnInstance(() -> assertEquals(Collections.emptySet(), ClusterMetadata.current().accordStaleReplicas.ids()));
cluster.get(1).runOnInstance(() -> assertEquals(Collections.emptySet(), ClusterMetadata.current().accordStaleReplicas.stale()));
cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1").asserts().failure().errorContains("it is not stale");
cluster.get(1).nodetoolResult("accord", "mark_rejoining", "4").asserts().failure().errorContains("not present in the directory");
@ -72,7 +72,7 @@ public class AccordNodetoolTest extends TestBaseImpl
cluster.get(1).nodetoolResult("accord", "mark_stale", "1", "2", "3").asserts().failure().errorContains("that would leave fewer than a quorum");
cluster.get(1).nodetoolResult("accord", "mark_stale", "1", "2").asserts().success();
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1), new Node.Id(2)), ClusterMetadata.current().accordStaleReplicas.ids()));
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1), new Node.Id(2)), ClusterMetadata.current().accordStaleReplicas.stale()));
cluster.get(1).nodetoolResult("accord", "describe").asserts().stdoutContains("Stale Replicas: 1,2");
// Reject the operation if a target node is already stale:
@ -89,11 +89,11 @@ public class AccordNodetoolTest extends TestBaseImpl
cluster.get(nodeIdToNode.get(2)).shutdown().get();
cluster.get(1).nodetoolResult("removenode", "2", "--force").asserts().success();
cluster.get(1).nodetoolResult("cms", "unregister", "2").asserts().success();
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids()));
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale()));
cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1", "3").asserts().failure().errorContains("it is not stale");
cluster.get(1).nodetoolResult("accord", "mark_rejoining", "1", "6").asserts().failure().errorContains("not present in the directory");
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.ids()));
cluster.get(1).runOnInstance(() -> assertEquals(ImmutableSet.of(new Node.Id(1)), ClusterMetadata.current().accordStaleReplicas.stale()));
}
}
}

215
test/distributed/org/apache/cassandra/distributed/test/accord/AccordRecoverFromAvailabilityLossTest.java Normal file
View File

@ -0,0 +1,215 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.junit.Test;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.topology.TopologyManager;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordOperations;
import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.FBUtilities;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.service.accord.AccordService.getBlocking;
public class AccordRecoverFromAvailabilityLossTest extends AccordBootstrapTestBase
{
private static void setConfig(IInstanceConfig config)
{
config.set("accord.command_store_shard_count", 2)
.set("accord.queue_shard_count", 2)
.set("accord.shard_durability_cycle", "20s")
.set("accord.shard_durability_target_splits", "1")
.set("accord.retry_syncpoint", "1s*attempts")
.set("accord.retry_durability", "1s*attempts")
.with(NETWORK, GOSSIP);
}
@Test
public void replaceWithAvailabilityLossTest() throws Throwable
{
int originalNodeCount = 3;
int expandedNodeCount = originalNodeCount + 1;
try (Cluster cluster = Cluster.build().withNodes(originalNodeCount)
.withoutVNodes()
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0"))
.withConfig(AccordRecoverFromAvailabilityLossTest::setConfig)
.start())
{
cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().reconfigureCMS(ReplicationParams.simpleMeta(3, ClusterMetadata.current().directory.knownDatacenters())));
cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':2}");
cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'");
awaitMaxEpochReadyToRead(cluster);
int removeIdx = 3;
int removeId = cluster.get(removeIdx).callOnInstance(() -> ClusterMetadata.current().myNodeId().id());
for (int key = 0; key < 50; key++)
{
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
" END IF\n" +
"COMMIT TRANSACTION";
AccordTestBase.executeWithRetry(cluster, query);
}
FBUtilities.waitOnFuture(cluster.get(removeIdx).shutdown(false));
{
List<java.util.concurrent.Future<?>> results = new ArrayList<>();
for (int key = 50; key < 100; key++)
{
String query = "BEGIN TRANSACTION\n" +
" LET row1 = (SELECT * FROM ks.tbl WHERE k = " + key + " AND c = 0);\n" +
" SELECT row1.v;\n" +
" IF row1 IS NULL THEN\n" +
" INSERT INTO ks.tbl (k, c, v) VALUES (" + key + ", " + key + ", " + key + ");\n" +
" END IF\n" +
"COMMIT TRANSACTION";
results.add(cluster.coordinator(2).asyncExecuteWithResult(query, ConsistencyLevel.SERIAL));
}
try { FBUtilities.waitOnFutures(results); }
catch (Throwable t) {}
}
Future<?> future = cluster.get(2).asyncAcceptsOnInstance((Integer id) -> {
while (true)
{
try
{
AccordOperations.instance.accordMarkHardRemoved(Set.of(new NodeId(id)), false);
break;
}
catch (Throwable t)
{
try
{
Thread.sleep(1000);
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
}
}).apply(removeId);
ClusterUtils.replaceHostAndStart(cluster, cluster.get(removeIdx), (i1, i2) -> {}, AccordRecoverFromAvailabilityLossTest::setConfig);
future.get();
awaitMaxEpochReadyToRead(cluster);
cluster.get(4).runOnInstance(() -> {
List<Range<Token>> ranges = StorageService.instance.getLocalRanges("ks");
TopologyManager topologyManager = service().node().topology();
for (long epoch = topologyManager.minEpoch() ; epoch <= topologyManager.epoch() ; ++epoch)
{
CountDownLatch latch = new CountDownLatch(1);
topologyManager.epochReady(epoch).data.invokeIfSuccess(latch::countDown);
while (true)
{
try
{
if (latch.await(1L, TimeUnit.SECONDS))
break;
}
catch (InterruptedException e)
{
throw new RuntimeException(e);
}
}
}
for (int key = 0; key < 100; key++)
{
UntypedResultSet result = QueryProcessor.executeInternal("SELECT * FROM ks.tbl WHERE k=?", key);
PartitionKey partitionKey = pk(key, "ks", "tbl");
if (ranges.stream().anyMatch(range -> range.contains(partitionKey.token())))
{
UntypedResultSet.Row row;
if (key < 50) row = getOnlyElement(result);
else try { row = getOnlyElement(result); } catch (NoSuchElementException e) { continue; }
Assert.assertEquals(key, row.getInt("c"));
Assert.assertEquals(key, row.getInt("v"));
getBlocking(service().node().commandStores().forEach("Test", RoutingKeys.of(partitionKey.toUnseekable()), Long.MIN_VALUE, Long.MAX_VALUE, safeStore -> {
if (safeStore.ranges().currentRanges().contains(partitionKey))
{
AccordSafeCommandStore ss = (AccordSafeCommandStore) safeStore;
Assert.assertFalse(ss.bootstrapBeganAt().isEmpty());
Assert.assertFalse(ss.safeToReadAt().isEmpty());
Assert.assertEquals(1, ss.bootstrapBeganAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
Assert.assertEquals(1, ss.safeToReadAt().entrySet().stream()
.filter(entry -> entry.getValue().contains(partitionKey))
.map(entry -> {
Assert.assertTrue(entry.getKey().compareTo(Timestamp.NONE) > 0);
return entry;
}).count());
}
}));
}
else
{
Assert.assertTrue(result.isEmpty());
}
}
});
}
}
}

153
test/distributed/org/apache/cassandra/distributed/test/accord/journal/AccordJournalReplayTest.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.accord.journal;
import org.junit.Test;
import accord.api.Journal;
import accord.coordinate.CoordinateSyncPoint;
import accord.coordinate.ExecuteSyncPoint;
import accord.local.Command;
import accord.local.Node;
import accord.local.RedundantBefore;
import accord.local.RedundantStatus;
import accord.local.StoreParticipants;
import accord.primitives.Ballot;
import accord.primitives.Deps;
import accord.primitives.FullRoute;
import accord.primitives.KeyDeps;
import accord.primitives.Keys;
import accord.primitives.RangeDeps;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.RoutingKeys;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
import accord.primitives.SyncPoint;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.ImmutableBitSet;
import accord.utils.LargeBitSet;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.Util.spinUntilTrue;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
public class AccordJournalReplayTest extends TestBaseImpl
{
private static DecoratedKey dk(int key)
{
IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
return partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
private static PartitionKey pk(int key, String keyspace, String table)
{
TableId tid = Schema.instance.getTableMetadata(keyspace, table).id;
return new PartitionKey(tid, dk(key));
}
@Test
public void replayCommandWithOnlyDurableSyncPointDependency() throws Throwable
{
try (Cluster cluster = Cluster.build().withNodes(1)
.withoutVNodes()
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(1))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(1, "dc0", "rack0"))
.withConfig(config -> config.set("accord.command_store_shard_count", 2)
.set("accord.queue_shard_count", 2)
.set("accord.shard_durability_cycle", "20s")
.set("accord.shard_durability_target_splits", "1")
.set("accord.retry_syncpoint", "1s*attempts")
.set("accord.retry_durability", "1s*attempts")
.with(NETWORK, GOSSIP))
.start())
{
cluster.schemaChange("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':1}");
cluster.schemaChange("CREATE TABLE ks.tbl (k int, c int, v int, primary key(k, c)) WITH transactional_mode='full'");
String txnIdStr = cluster.get(1).callOnInstance(() -> {
AccordService service = (AccordService) AccordService.instance();
PartitionKey key = pk(1, "ks", "tbl");
Node node = service.node();
Txn txn = AccordTestUtils.createTxn("BEGIN TRANSACTION\n" +
"INSERT INTO ks.tbl (k, c, v) VALUES (?, ?, ?);\n" +
"COMMIT TRANSACTION", 1, 1, 1);
Txn syncPointTxn = node.agent().emptySystemTxn(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range);
TxnId syncPointId = node.nextTxnId(syncPointTxn);
TxnId txnId = node.nextTxnId(txn);
FullRoute<?> route = node.computeRoute(txnId, txn.keys());
AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().unsafeForKey(key.toUnseekable());
Deps deps = new Deps(KeyDeps.NONE, RangeDeps.SerializerSupport.create(new accord.primitives.Range[] { key.asRange() }, new TxnId[] { syncPointId }, new int[] { 2, 0 }, new int[] { 2, 0 }));
Command.WaitingOn waitingOn; {
LargeBitSet waitingOnBits = new LargeBitSet(1);
waitingOnBits.set(0);
waitingOn = new Command.WaitingOn(RoutingKeys.EMPTY, deps.rangeDeps, new ImmutableBitSet(waitingOnBits), null);
}
Writes writes = new Writes(txnId, txnId, Keys.of(key), null);
Command command = Command.Executed.executed(txnId, SaveStatus.PreApplied, Status.Durability.NotDurable, StoreParticipants.execute(commandStore.unsafeGetRangesForEpoch(), route, txnId, txnId.epoch()), Ballot.ZERO, txnId, txn.intersecting(route, true), deps.intersecting(route), Ballot.ZERO, waitingOn, writes, ResultSerializers.APPLIED);
commandStore.journal.saveCommand(commandStore.id(), new Journal.CommandUpdate(null, command), () -> {});
SyncPoint<accord.primitives.Range> syncPoint = AccordService.getBlocking(CoordinateSyncPoint.exclusive(node, syncPointId, Ranges.of(key.asRange())));
AccordService.getBlocking(ExecuteSyncPoint.coordinate(node, syncPoint, 1).onQuorum());
Keyspace.open("ks").getColumnFamilyStore("tbl").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
Keyspace.open("system_accord").getColumnFamilyStore("commands_for_key").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
spinUntilTrue(() -> {
RedundantBefore.Bounds bounds = commandStore.unsafeGetRedundantBefore().get(key);
return bounds.maxBound(RedundantStatus.Property.LOCALLY_DURABLE_TO_DATA_STORE).equals(syncPointId)
&& bounds.maxBound(RedundantStatus.Property.LOCALLY_DURABLE_TO_COMMAND_STORE).equals(syncPointId);
});
return txnId.toString();
});
cluster.get(1).shutdown(false);
cluster.get(1).startup();
cluster.get(1).runOnInstance(() -> {
AccordService service = (AccordService) AccordService.instance();
PartitionKey key = pk(1, "ks", "tbl");
AccordCommandStore commandStore = (AccordCommandStore) service.node().commandStores().unsafeForKey(key.toUnseekable());
TxnId txnId = TxnId.parse(txnIdStr);
Command command = commandStore.loadCommand(txnId);
Assertions.assertThat(command.saveStatus()).isEqualTo(SaveStatus.Applied);
});
}
}
}

2
test/distributed/org/apache/cassandra/distributed/test/hostreplacement/NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest.java
View File

@ -30,6 +30,7 @@ import org.junit.Test;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.Constants;
import org.apache.cassandra.distributed.api.Feature;
@ -53,6 +54,7 @@ public class NodeCannotJoinAsHibernatingNodeWithoutReplaceAddressTest extends Te
@Test
public void test() throws IOException, InterruptedException
{
CassandraRelevantProperties.DTEST_ACCORD_ENABLED.setBoolean(false); // when added, accord could not replace same address safely
TokenSupplier even = TokenSupplier.evenlyDistributedTokens(2);
try (Cluster cluster = init(Cluster.build(2)
.withConfig(c -> c.with(Feature.values()).set(Constants.KEY_DTEST_API_STARTUP_FAILURE_AS_SHUTDOWN, false))

10
test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTest.java
View File

@ -91,7 +91,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
if (!prediction.state.get().isWriteTargetFor(token, prediction.node(6).matcher))
continue;
simulatedCluster.waitForQuiescense();
simulatedCluster.waitForQuiescence();
List<Replica> replicas = simulatedCluster.state.get().writePlacementsFor(token);
// At most 2 replicas should respond, so that when the pending node is added, results would be insufficient for recomputed blockFor
BooleanSupplier shouldRespond = atMostResponses(simulatedCluster.state.get().isWriteTargetFor(token, simulatedCluster.node(1).matcher) ? 1 : 2);
@ -121,7 +121,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
.prepareJoin()
.startJoin();
simulatedCluster.waitForQuiescense();
simulatedCluster.waitForQuiescence();
waiting.forEach(WaitingAction::resume);
@ -165,7 +165,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
!simulatedCluster.state.get().isReadReplicaFor(token(pk), simulatedCluster.node(1).matcher))
continue;
simulatedCluster.waitForQuiescense();
simulatedCluster.waitForQuiescence();
List<Replica> replicas = simulatedCluster.state.get().readReplicasFor(token(pk));
Function<Integer, BooleanSupplier> shouldRespond = respondFrom(1, 4);
@ -185,7 +185,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
.midLeave()
.finishLeave();
simulatedCluster.waitForQuiescense();
simulatedCluster.waitForQuiescence();
waiting.forEach(WaitingAction::resume);
@ -224,7 +224,7 @@ public class CoordinatorPathTest extends CoordinatorPathTestBase
.prepareJoin()
.startJoin();
simulatedCluster.waitForQuiescense();
simulatedCluster.waitForQuiescence();
AtomicInteger reads = new AtomicInteger();
AtomicInteger writes = new AtomicInteger();

2
test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java
View File

@ -831,7 +831,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
return nodes.values().stream().filter(predicate);
}
public void waitForQuiescense()
public void waitForQuiescence()
{
Epoch waitFor = ClusterMetadataService.instance().log().waitForHighestConsecutive().epoch;
realCluster.get(1).acceptsOnInstance((Long epoch) -> {

4
test/distributed/org/apache/cassandra/fuzz/topology/AccordTopologyMixupTest.java
View File

@ -383,14 +383,14 @@ public class AccordTopologyMixupTest extends TopologyMixupTestBase<AccordTopolog
});
}
@Override
public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Throwable failure)
public void onFailedBootstrap(int attempts, String phase, Ranges ranges, Runnable retry, Runnable fail, Throwable failure)
{
if (failure instanceof Exhausted)
{
Exhausted e = (Exhausted) failure;
SharedState.debugTxn(self.id, "Bootstrap#" + phase, e.txnId().toString());
}
super.onFailedBootstrap(attempts, phase, ranges, retry, failure);
super.onFailedBootstrap(attempts, phase, ranges, retry, fail, failure);
}
}
}

76
test/unit/org/apache/cassandra/db/virtual/AccordDebugKeyspaceTest.java
View File

@ -33,8 +33,6 @@ import java.util.function.BiPredicate;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.ProtocolModifiers;
import accord.messages.NoWaitRequest;
@ -86,8 +84,6 @@ import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
public class AccordDebugKeyspaceTest extends CQLTester
{
private static final Logger logger = LoggerFactory.getLogger(AccordDebugKeyspaceTest.class);
private static final String QUERY_TXN_BLOCKED_BY =
String.format("SELECT * FROM %s.%s WHERE txn_id=?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.TXN_BLOCKED_BY);
@ -170,22 +166,22 @@ public class AccordDebugKeyspaceTest extends CQLTester
String.format("DELETE FROM %s.%s WHERE node_id = ? AND txn_id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.TXN_TRACES);
private static final String QUERY_REDUNDANT_BEFORE =
String.format("SELECT * FROM %s.%s", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
String.format("SELECT * FROM %s.%s where table_id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_REMOTE =
String.format("SELECT * FROM %s.%s WHERE node_id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
String.format("SELECT * FROM %s.%s WHERE node_id = ? AND table_id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ =
String.format("SELECT * FROM %s.%s WHERE quorum_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ =
String.format("SELECT * FROM %s.%s WHERE table_id = ? AND quorum_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE =
String.format("SELECT * FROM %s.%s WHERE node_id = ? AND quorum_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE =
String.format("SELECT * FROM %s.%s WHERE node_id = ? AND table_id = ? AND quorum_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ =
String.format("SELECT * FROM %s.%s WHERE shard_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ =
String.format("SELECT * FROM %s.%s WHERE table_id = ? AND shard_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE =
String.format("SELECT * FROM %s.%s WHERE node_id = ? AND shard_applied >= ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE =
String.format("SELECT * FROM %s.%s WHERE node_id = ? AND table_id = ? AND shard_applied", SchemaConstants.VIRTUAL_ACCORD_DEBUG_REMOTE, AccordDebugKeyspace.REDUNDANT_BEFORE);
private static final String SET_PATTERN_TRACE =
String.format("UPDATE %s.%s SET bucket_mode = ?, bucket_seen = ?, bucket_size = ?, chance = ?, if_intersects = ?, if_kind = ?, on_failure = ?, on_new = ?, trace_bucket_mode = ?, trace_bucket_size = ?, trace_bucket_sub_size = ?, trace_events = ? WHERE id = ?", SchemaConstants.VIRTUAL_ACCORD_DEBUG, AccordDebugKeyspace.TXN_PATTERN_TRACE);
@ -424,16 +420,40 @@ public class AccordDebugKeyspaceTest extends CQLTester
safeStore.commandStore().markShardDurable(safeStore, syncId2, ranges2, HasOutcome.Quorum);
}));
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE).size()).isGreaterThan(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ, syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ, syncId2.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ, syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ, syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_REMOTE, nodeId).size()).isGreaterThan(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE, nodeId, syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_GEQ_REMOTE, nodeId, syncId2.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE, nodeId, syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_GEQ_REMOTE, nodeId, syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE, tableId.toString()).size()).isGreaterThan(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " >= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " >= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " >= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " >= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " > ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " > ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " > ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " > ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " <= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " <= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " <= ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " <= ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " < ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ + " < ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " < ?", tableId.toString(), syncId1.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ + " < ?", tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_REMOTE, nodeId, tableId.toString()).size()).isGreaterThan(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " >= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(1);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " > ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(0);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " <= ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(2);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_QUORUM_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId1.toString()).size()).isEqualTo(3);
Assertions.assertThat(execute(QUERY_REDUNDANT_BEFORE_FILTER_SHARD_APPLIED_INEQ_REMOTE + " < ?", nodeId, tableId.toString(), syncId2.toString()).size()).isEqualTo(4);
}
@Test
@ -453,7 +473,7 @@ public class AccordDebugKeyspaceTest extends CQLTester
}
@Test
public void completedTxn() throws ExecutionException, InterruptedException
public void completedTxn()
{
String tableName = createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode = 'full'");
AccordService accord = accord();
@ -470,9 +490,9 @@ public class AccordDebugKeyspaceTest extends CQLTester
getBlocking(accord.node().coordinate(id, txn));
filter.apply.awaitThrowUncheckedOnInterrupt();
spinUntilSuccess(() -> assertRows(execute(QUERY_TXN_BLOCKED_BY, id.toString()),
row(id.toString(), anyInt(), 0, "", "", any(), anyOf(SaveStatus.ReadyToExecute.name(), SaveStatus.Applying.name(), SaveStatus.Applied.name()))));
assertRows(execute(QUERY_TXN, id.toString()), row(id.toString(), anyOf("Applied", "Applying")));
assertRows(execute(QUERY_TXN_REMOTE, nodeId, id.toString()), row(id.toString(), anyOf("Applied", "Applying")));
row(id.toString(), anyInt(), 0, "", "", any(), "Applied")));
assertRows(execute(QUERY_TXN, id.toString()), row(id.toString(), "Applied"));
assertRows(execute(QUERY_TXN_REMOTE, nodeId, id.toString()), row(id.toString(), "Applied"));
assertRows(execute(QUERY_JOURNAL, id.toString()), row(id.toString(), "PreAccepted"), row(id.toString(), "Applying"), row(id.toString(), "Applied"), row(id.toString(), null));
assertRows(execute(QUERY_JOURNAL_REMOTE, nodeId, id.toString()), row(id.toString(), "PreAccepted"), row(id.toString(), "Applying"), row(id.toString(), "Applied"), row(id.toString(), null));
assertRows(execute(QUERY_COMMANDS_FOR_KEY, keyStr), row(id.toString(), "APPLIED_DURABLE"));

14
test/unit/org/apache/cassandra/service/accord/AccordConfigurationServiceTest.java
View File

@ -61,6 +61,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.service.accord.EndpointMapping.Updateable;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ValidatingClusterMetadataService;
@ -70,7 +71,6 @@ import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.MockFailureDetector;
import org.apache.cassandra.utils.concurrent.Future;
import static accord.impl.AbstractConfigurationServiceTest.TestListener;
@ -170,23 +170,23 @@ public class AccordConfigurationServiceTest
try
{
journal = initJournal();
AccordConfigurationService service = new AccordConfigurationService(ID1, new AccordAgent(), new Messaging(), new MockFailureDetector(), ScheduledExecutors.scheduledTasks);
AccordConfigurationService service = new AccordConfigurationService(ID1, new AccordAgent(), new Updateable(), new Messaging(), ScheduledExecutors.scheduledTasks);
AccordJournal journal_ = journal;
TestListener listener = new TestListener(service, true)
{
@Override
public AsyncResult<Void> onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync)
public AsyncResult<Void> onTopologyUpdate(Topology topology)
{
// Fake journal save
journal_.saveTopology(new TopologyUpdate(new Int2ObjectHashMap<>(), topology), () -> {});
return super.onTopologyUpdate(topology, isLoad, startSync);
return super.onTopologyUpdate(topology);
}
};
service.registerListener(listener);
service.start();
Topology topology1 = createTopology(cms);
service.updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1));
((Updateable)service.endpointMapper()).updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1));
service.reportTopology(topology1);
service.receiveRemoteSyncComplete(ID1, 1);
service.receiveRemoteSyncComplete(ID2, 1);
@ -199,8 +199,8 @@ public class AccordConfigurationServiceTest
Topology topology3 = createTopology(cms);
service.reportTopology(topology3);
AccordConfigurationService loaded = new AccordConfigurationService(ID1, new AccordAgent(), new Messaging(), new MockFailureDetector(), ScheduledExecutors.scheduledTasks);
loaded.updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1));
AccordConfigurationService loaded = new AccordConfigurationService(ID1, new AccordAgent(), new Updateable(), new Messaging(), ScheduledExecutors.scheduledTasks);
((EndpointMapping.Updateable)loaded.endpointMapper()).updateMapping(mappingForEpoch(cms.metadata().epoch.getEpoch() + 1));
listener = new AbstractConfigurationServiceTest.TestListener(loaded, true);
loaded.registerListener(listener);
journal_.closeCurrentSegmentForTestingIfNonEmpty();

4
test/unit/org/apache/cassandra/service/accord/AccordMessageSinkTest.java
View File

@ -87,7 +87,7 @@ public class AccordMessageSinkTest
checkRequestReplies(request,
new AbstractFetchCoordinator.FetchResponse(null, null, id),
CommitOrReadNack.Insufficient);
CommitOrReadNack.InsufficientAndWaiting);
}
@ -98,7 +98,7 @@ public class AccordMessageSinkTest
Request request = new ReadTxnData(node, topologies, txnId, topology.ranges(), null, null, txnId.epoch());
checkRequestReplies(request,
new ReadData.ReadOk(null, null, 0),
CommitOrReadNack.Insufficient);
CommitOrReadNack.InsufficientAndWaiting);
}
private static void checkRequestReplies(Request request, Reply... replies)

5
test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java
View File

@ -27,6 +27,7 @@ import accord.local.Node;
import accord.utils.AccordGens;
import accord.utils.Gen;
import accord.utils.Gens;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.tcm.Epoch;
@ -52,8 +53,8 @@ public class AccordStaleReplicasTest
qt().check(rs -> {
Epoch epoch = epochGen.next(rs);
Set<Node.Id> nodes = nodesGen.next(rs);
AsymmetricMetadataSerializers.testSerde(buffer, AccordStaleReplicas.serializer, new AccordStaleReplicas(nodes, epoch), Version.MIN_ACCORD_VERSION);
SortedArrayList<Node.Id> nodes = SortedArrayList.copyUnsorted(nodesGen.next(rs), Node.Id[]::new);
AsymmetricMetadataSerializers.testSerde(buffer, AccordStaleReplicas.serializer, new AccordStaleReplicas(nodes, SortedArrayList.ofSorted(), epoch), Version.MIN_ACCORD_VERSION);
});
}
}

156
test/unit/org/apache/cassandra/service/accord/AccordSyncPropagatorTest.java
View File

@ -33,9 +33,10 @@ import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.junit.BeforeClass;
import org.junit.Test;
@ -56,6 +57,9 @@ import accord.utils.AccordGens;
import accord.utils.Gen;
import accord.utils.Gens;
import accord.utils.RandomSource;
import accord.utils.SortedArrays.SortedArrayList;
import accord.utils.SortedList;
import accord.utils.SortedListSet;
import org.apache.cassandra.concurrent.AdaptingScheduledExecutorPlus;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.config.DatabaseDescriptor;
@ -63,8 +67,6 @@ import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.HeartBeatState;
import org.apache.cassandra.gms.IFailureDetectionEventListener;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.ConnectionType;
import org.apache.cassandra.net.Message;
@ -100,8 +102,8 @@ public class AccordSyncPropagatorTest
// so when instances are created here they are added to gossip to trick the membership check...
Gossiper.instance.clearUnsafe();
List<Node.Id> nodes = nodesGen.next(rs);
Set<Node.Id> nodesAsSet = ImmutableSet.copyOf(nodes);
SortedList<Node.Id> nodes = SortedArrayList.copyUnsorted(nodesGen.next(rs), Node.Id[]::new);
SortedListSet<Node.Id> nodesAsSet = SortedListSet.allOf(nodes);
List<Throwable> failures = new ArrayList<>();
RandomDelayQueue delayQueue = new RandomDelayQueue.Factory(rs).get();
@ -123,13 +125,16 @@ public class AccordSyncPropagatorTest
allRanges.put(epoch, ranges);
scheduler.schedule(() -> {
for (Node.Id nodeId : nodes)
cluster.node(nodeId).propagator.reportSyncComplete(epoch, nodes, nodeId);
{
cluster.node(nodeId).configurationService.receiveRemoteSyncComplete(nodeId, epoch);
cluster.node(nodeId).propagator.reportCoordinationReady(epoch, nodes, nodeId);
}
for (int j = 0, attempts = rs.nextInt(1, 4); j < attempts; j++)
{
for (Range range : ranges)
{
Cluster.Instace inst = cluster.node(choose(rs, nodes));
Cluster.Instance inst = cluster.node(choose(rs, nodesAsSet));
scheduler.schedule(() -> {
Ranges subrange = Ranges.of(range);
inst.propagator.reportClosed(epoch, nodes, subrange);
@ -159,7 +164,7 @@ public class AccordSyncPropagatorTest
if (hasPending(cluster))
throw new AssertionError("Unable to make progress: pending syncs on \n" + cluster.instances.values().stream().filter(i -> i.propagator.hasPending()).map(i -> i.propagator.toString()).collect(Collectors.joining("\n")));
for (Cluster.Instace inst : cluster.instances.values())
for (Cluster.Instance inst : cluster.instances.values())
{
Cluster.ConfigService cs = inst.configurationService;
assertSetsEqual(cs.completedEpochs, allRanges.keySet(), "completedEpochs %s", inst.id);
@ -204,7 +209,7 @@ public class AccordSyncPropagatorTest
private static class Cluster implements AccordEndpointMapper
{
private final ImmutableBiMap<Node.Id, InetAddressAndPort> nodeToAddress;
private final ImmutableMap<Node.Id, Instace> instances;
private final ImmutableMap<Node.Id, Instance> instances;
private final RandomSource rs;
private final ScheduledExecutorPlus scheduler;
@ -215,15 +220,15 @@ public class AccordSyncPropagatorTest
this.rs = rs;
this.scheduler = scheduler;
ImmutableBiMap.Builder<Node.Id, InetAddressAndPort> nodeToAddress = ImmutableBiMap.builder();
ImmutableMap.Builder<Node.Id, Instace> instances = ImmutableMap.builder();
ImmutableMap.Builder<Node.Id, Instance> instances = ImmutableMap.builder();
for (Node.Id id : nodes)
{
InetAddressAndPort address = addressFromInt(id.id);
nodeToAddress.put(id, address);
ConfigService cs = new ConfigService(id);
ConfigService cs = new ConfigService(id, nodes);
Sink sink = new Sink(id);
IFailureDetector fd = new FailureDetector(address);
instances.put(id, new Instace(id, address, cs, sink, fd, cs, new AccordSyncPropagator(id, Cluster.this, sink, fd, scheduler, cs)));
FailureWrapper fw = new FailureWrapper(Cluster.this, id);
instances.put(id, new Instance(id, cs, sink, new AccordSyncPropagator(id, fw, sink, scheduler, cs)));
Gossiper.instance.endpointStateMap.put(address, new EndpointState(HeartBeatState.empty()));
}
this.nodeToAddress = nodeToAddress.build();
@ -244,31 +249,43 @@ public class AccordSyncPropagatorTest
}
}
public Cluster.Instace node(Node.Id id)
public Instance node(Node.Id id)
{
Instace instace = instances.get(id);
if (instace == null)
Instance instance = instances.get(id);
if (instance == null)
throw new NullPointerException("Unknown id: " + id);
return instace;
return instance;
}
public Cluster.Instace node(InetAddressAndPort address)
public Instance node(InetAddressAndPort address)
{
return node(mappedId(address));
return node(mappedIdOrNull(address));
}
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object ignore)
{
return nodeToAddress.inverse().get(endpoint);
}
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object ignore)
{
return nodeToAddress.get(id);
}
@Override
public Map<Node.Id, Long> removedNodes()
{
return Map.of();
}
@Override
public NodeStatus nodeStatus(Node.Id id)
{
throw new UnsupportedOperationException();
}
private enum Action
{
DELIVER, TIMEOUT, ERROR
@ -309,7 +326,7 @@ public class AccordSyncPropagatorTest
throw new IllegalStateException("Unknown action: " + action);
}
callbacks.put(message.id(), cb);
scheduler.schedule(() -> AccordService.receive(this, node(to).configurationService, (Message<AccordSyncPropagator.Notification>) message.withFrom(mappedEndpoint(from))), 500, TimeUnit.MILLISECONDS);
scheduler.schedule(() -> AccordService.receive(this, node(to).configurationService, (Message<AccordSyncPropagator.Notification>) message.withFrom(mappedEndpointOrNull(from))), 500, TimeUnit.MILLISECONDS);
scheduler.schedule(() -> {
RequestCallback<?> removed = callbacks.remove(message.id());
if (removed != null)
@ -355,72 +372,60 @@ public class AccordSyncPropagatorTest
}
}
private class FailureDetector implements IFailureDetector
private class FailureWrapper implements AccordEndpointMapper
{
private final InetAddressAndPort self;
private final Map<InetAddressAndPort, Gen<Boolean>> nodeRuns = new HashMap<>();
private final AccordEndpointMapper wrapped;
private final Node.Id self;
private final Map<Node.Id, Gen<Boolean>> nodeRuns = new HashMap<>();
private FailureDetector(InetAddressAndPort self)
private FailureWrapper(AccordEndpointMapper wrapped, Node.Id self)
{
this.wrapped = wrapped;
this.self = self;
}
@Nullable
@Override
public boolean isAlive(InetAddressAndPort ep)
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped)
{
if (self.equals(ep)) return true;
return wrapped.mappedIdOrNull(endpoint, logIdentityIfUnmapped);
}
return !nodeRuns.computeIfAbsent(ep, ignore -> Gens.bools().biasedRepeatingRuns(.01, rs.nextInt(3, 15))).next(rs);
@Nullable
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped)
{
return wrapped.mappedEndpointOrNull(id, logIdentityIfUnmapped);
}
@Override
public void interpret(InetAddressAndPort ep)
public NodeStatus nodeStatus(Node.Id id)
{
throw new UnsupportedOperationException();
if (self.equals(id)) return NodeStatus.HEALTHY;
return !nodeRuns.computeIfAbsent(id, ignore -> Gens.bools().biasedRepeatingRuns(.01, rs.nextInt(3, 15))).next(rs) ? NodeStatus.HEALTHY : NodeStatus.UNHEALTHY;
}
@Override
public void report(InetAddressAndPort ep)
public Map<Node.Id, Long> removedNodes()
{
throw new UnsupportedOperationException();
}
@Override
public void remove(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
@Override
public void forceConviction(InetAddressAndPort ep)
{
throw new UnsupportedOperationException();
}
@Override
public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener)
{
throw new UnsupportedOperationException();
}
@Override
public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener)
{
throw new UnsupportedOperationException();
return Map.of();
}
}
private class ConfigService extends AbstractTestConfigurationService implements AccordSyncPropagator.Listener
{
private final List<Node.Id> nodes;
private final Map<Long, Set<Node.Id>> syncCompletes = new HashMap<>();
private final Map<Long, Set<Node.Id>> endpointAcks = new HashMap<>();
private final NavigableSet<Long> completedEpochs = Collections.synchronizedNavigableSet(new TreeSet<>());
private final Map<Long, Ranges> closed = new HashMap<>();
private final Map<Long, Ranges> redundant = new HashMap<>();
private ConfigService(Node.Id node)
private ConfigService(Node.Id node, List<Node.Id> nodes)
{
super(node, new AccordAgent());
this.nodes = nodes;
}
@Override
@ -436,10 +441,9 @@ public class AccordSyncPropagatorTest
}
@Override
protected void onReadyToCoordinate(Topology topology, boolean startSync)
protected void onReadyToCoordinate(Topology topology)
{
Set<Node.Id> notify = topology.nodes().stream().filter(i -> !localId.equals(i)).collect(Collectors.toSet());
instances.get(localId).propagator.reportSyncComplete(topology.epoch(), notify, localId);
instances.get(localId).propagator.reportCoordinationReady(topology.epoch(), topology.nodes(), localId);
}
@Override
@ -464,16 +468,15 @@ public class AccordSyncPropagatorTest
@Override
public void onEndpointAck(Node.Id id, long epoch)
{
endpointAcks.computeIfAbsent(epoch, ignore -> new HashSet<>()).add(id);
Set<Node.Id> acks = endpointAcks.computeIfAbsent(epoch, ignore -> new HashSet<>());
if (acks.add(id) && acks.containsAll(nodes))
completedEpochs.add(epoch);
}
@Override
public void onComplete(long epoch)
public void onEndpointNack(Node.Id id, long epoch)
{
completedEpochs.add(epoch);
// TODO why do we see multiple calls?
// if (!completedEpochs.add(epoch))
// throw new IllegalStateException("Completed epoch " + epoch + " multiple times");
onEndpointAck(id, epoch);
}
@Override
@ -491,30 +494,21 @@ public class AccordSyncPropagatorTest
}
}
public class Instace
public class Instance
{
private final Node.Id id;
private final InetAddressAndPort address;
private final ConfigService configurationService;
private final Sink messagingService;
private final IFailureDetector failureDetector;
private final AccordSyncPropagator.Listener listener;
private final AccordSyncPropagator propagator;
private Instace(Node.Id id,
InetAddressAndPort address,
ConfigService configurationService,
Sink messagingService,
IFailureDetector failureDetector,
AccordSyncPropagator.Listener listener,
AccordSyncPropagator propagator)
private Instance(Node.Id id,
ConfigService configurationService,
Sink messagingService,
AccordSyncPropagator propagator)
{
this.id = id;
this.address = address;
this.configurationService = configurationService;
this.messagingService = messagingService;
this.failureDetector = failureDetector;
this.listener = listener;
this.propagator = propagator;
}
}

5
test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
View File

@ -382,6 +382,7 @@ public class AccordTestUtils
@Override public TopologyManager topology() { throw new UnsupportedOperationException(); }
@Override public long currentStamp() { return stamp; }
@Override public void updateStamp() {++stamp;}
@Override public boolean isReplaying() { return false; }
};
AccordAgent agent = new AccordAgent();
@ -397,7 +398,7 @@ public class AccordTestUtils
holder.add(1, new CommandStores.RangesForEpoch(1, ranges), ranges);
AccordCommandStore result = new AccordCommandStore(0, time, agent, null,
cs -> new NoOpProgressLog(),
cs -> new DefaultLocalListeners(new NoOpRemoteListeners(), new NoOpNotifySink()),
cs -> new DefaultLocalListeners(null, new NoOpRemoteListeners(), new NoOpNotifySink()),
holder, journal, executor);
result.unsafeUpdateRangesForEpoch();
return result;
@ -409,7 +410,7 @@ public class AccordTestUtils
TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
TokenRange range = TokenRange.fullRange(metadata.id, metadata.partitioner);
Node.Id node = new Id(1);
Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node), Collections.emptySet()));
Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node)));
AccordCommandStore store = createAccordCommandStore(node, now, topology);
store.execute((PreLoadContext.Empty)()->"Test", safeStore -> ((AccordCommandStore)safeStore.commandStore()).executor().cacheUnsafe().setCapacity(1 << 20));
return store;

2
test/unit/org/apache/cassandra/service/accord/EndpointMappingTest.java
View File

@ -35,7 +35,7 @@ public class EndpointMappingTest
{
qt().forAll(CassandraGenerators.INET_ADDRESS_AND_PORT_GEN, SourceDSL.integers().between(1, Integer.MAX_VALUE).map(Node.Id::new)).checkAssert((endpoint, id) -> {
EndpointMapping mapping = EndpointMapping.builder(1).add(endpoint, id).build();
Assertions.assertThat(mapping.mappedEndpoint(id)).isEqualTo(endpoint);
Assertions.assertThat(mapping.mappedEndpointOrNull(id)).isEqualTo(endpoint);
});
}
}

54
test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java
View File

@ -42,6 +42,8 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import javax.annotation.Nullable;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.slf4j.Logger;
@ -71,8 +73,6 @@ import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.gms.IFailureDetectionEventListener;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
@ -128,7 +128,7 @@ public class EpochSyncTest
@Test
public void test()
{
stateful().withExamples(50).withSteps(500).check(commands(() -> Cluster::new)
stateful().withSeed(1).withExamples(50).withSteps(500).check(commands(() -> Cluster::new)
.destroyState(cluster -> {
finishPendingWork(cluster);
cluster.processAll();
@ -210,50 +210,32 @@ public class EpochSyncTest
private int nodeCounter = 0;
private final ValidatingClusterMetadataService cms = ValidatingClusterMetadataService.createAndRegister(NodeVersion.CURRENT_METADATA_VERSION);
private final IFailureDetector fd = new IFailureDetector()
class Mapper implements AccordEndpointMapper
{
@Override
public boolean isAlive(InetAddressAndPort ep)
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, @Nullable Object logIdentityIfUnmapped)
{
return instances.get(nodeId(ep)).status != Status.Removed;
return nodeId(endpoint);
}
@Override
public void interpret(InetAddressAndPort ep)
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, @Nullable Object logIdentityIfUnmapped)
{
return address(id);
}
@Override
public void report(InetAddressAndPort ep)
public Map<Node.Id, Long> removedNodes()
{
return Map.of();
}
@Override
public void remove(InetAddressAndPort ep)
public NodeStatus nodeStatus(Node.Id id)
{
return instances.get(id).status != Status.Removed ? NodeStatus.HEALTHY : NodeStatus.UNKNOWN;
}
@Override
public void forceConviction(InetAddressAndPort ep)
{
}
@Override
public void registerFailureDetectionEventListener(IFailureDetectionEventListener listener)
{
}
@Override
public void unregisterFailureDetectionEventListener(IFailureDetectionEventListener listener)
{
}
};
}
public Cluster(RandomSource rs)
{
@ -579,7 +561,7 @@ public class EpochSyncTest
ClusterMetadata.Transformer builder = cms.metadata().transformer();
Instance instance = new Instance(id, token, builder.epoch(), createMessaging(id), fd);
Instance instance = new Instance(id, token, builder.epoch(), createMessaging(id), new Mapper());
instances.put(id, instance);
tokens.add(token);
@ -675,7 +657,7 @@ public class EpochSyncTest
private final Epoch epoch;
private Status status = Status.Init;
Instance(Node.Id node, long token, Epoch epoch, SimulatedMessageDelivery messagingService, IFailureDetector failureDetector)
Instance(Node.Id node, long token, Epoch epoch, SimulatedMessageDelivery messagingService, AccordEndpointMapper mapper)
{
this.id = node;
this.token = token;
@ -683,11 +665,11 @@ public class EpochSyncTest
// TODO (review): Should there be a real scheduler here? Is it possible to adapt the Scheduler interface to scheduler used in this test?
TimeService time = TimeService.ofNonMonotonic(globalExecutor::currentTimeMillis, TimeUnit.MILLISECONDS);
this.topology = new TopologyManager(SizeOfIntersectionSorter.SUPPLIER, new TestAgent.RethrowAgent(), id, time, new DefaultTimeouts(time));
config = new AccordConfigurationService(node, new AccordAgent(), messagingService, failureDetector, scheduler);
config = new AccordConfigurationService(node, new AccordAgent(), mapper, messagingService, scheduler);
config.registerListener(new ConfigurationService.Listener()
{
@Override
public AsyncResult<Void> onTopologyUpdate(Topology topology, boolean isLoad, boolean startSync)
public AsyncResult<Void> onTopologyUpdate(Topology topology)
{
AsyncResult<Void> metadata = schedule(rs.nextInt(1, 10), TimeUnit.SECONDS, (Callable<Void>) () -> null).beginAsResult();
AsyncResult<Void> coordination = metadata.flatMap(ignore -> schedule(rs.nextInt(1, 10), TimeUnit.SECONDS, (Callable<Void>) () -> null).beginAsResult());
@ -699,7 +681,7 @@ public class EpochSyncTest
ready.coordinate.invokeIfSuccess(() -> topology().onEpochSyncComplete(id, topology.epoch()));
if (topology().minEpoch() == topology.epoch() && topology().epoch() != topology.epoch())
return ready.coordinate;
config.acknowledgeEpoch(ready, startSync);
config.acknowledgeEpoch(ready);
return ready.coordinate;
}

17
test/unit/org/apache/cassandra/service/accord/SimpleAccordEndpointMapper.java
View File

@ -21,6 +21,7 @@ package org.apache.cassandra.service.accord;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Map;
import accord.local.Node;
import org.apache.cassandra.locator.InetAddressAndPort;
@ -31,7 +32,7 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper
INSTANCE;
@Override
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint)
public Node.Id mappedIdOrNull(InetAddressAndPort endpoint, Object ignore)
{
if (endpoint.addressBytes.length != 4)
throw new IllegalArgumentException("Only IPV4 is allowed: given " + endpoint.toString(true));
@ -39,7 +40,7 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper
}
@Override
public InetAddressAndPort mappedEndpointOrNull(Node.Id id)
public InetAddressAndPort mappedEndpointOrNull(Node.Id id, Object ignore)
{
byte[] array = ByteBufferUtil.bytes(id.id).array();
try
@ -51,4 +52,16 @@ public enum SimpleAccordEndpointMapper implements AccordEndpointMapper
throw new AssertionError("Unable to convert " + id + " to an IPV4 address", e);
}
}
@Override
public Map<Node.Id, Long> removedNodes()
{
return Map.of();
}
@Override
public NodeStatus nodeStatus(Node.Id id)
{
return NodeStatus.HEALTHY;
}
}

10
test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
View File

@ -241,6 +241,12 @@ public class SimulatedAccordCommandStore implements AutoCloseable
{
++stamp;
}
@Override
public boolean isReplaying()
{
return false;
}
};
TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent()
@ -265,7 +271,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
agent,
null,
ignore -> new ProgressLog.NoOpProgressLog(),
cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
cs -> new DefaultLocalListeners(null, new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
{
@Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
@Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
@ -458,7 +464,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
{
TxnId txnId = nextTxnId(txn.kind(), txn.keys().domain());
Ballot ballot = Ballot.fromValues(storeService.epoch(), storeService.now(), nodeId);
BeginRecovery br = new BeginRecovery(nodeId, topologies, txnId, null, false, txn, route, ballot);
BeginRecovery br = new BeginRecovery(nodeId, topologies, txnId, null, 0, txn, route, ballot);
return Pair.create(txnId, processAsync(br, safe -> {
var reply = br.apply(safe);

2
test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java
View File

@ -288,7 +288,7 @@ public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester
});
var delay = preAcceptAsync.flatMap(ignore -> AsyncChains.chain(instance.unorderedScheduled, () -> {
Ballot ballot = Ballot.fromValues(instance.storeService.epoch(), instance.storeService.now(), nodeId);
return new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, null, false, txn, route, ballot);
return new BeginRecovery(nodeId, new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, instance.topology), txnId, null, 0, txn, route, ballot);
}).beginAsResult());
var recoverAsync = delay.flatMap(br -> instance.processAsync(br, safe -> {
var reply = br.apply(safe);

3
test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java
View File

@ -645,7 +645,7 @@ public class CommandsForKeySerializerTest
null,
null,
ignore -> new ProgressLog.NoOpProgressLog(),
ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE),
ignore -> new DefaultLocalListeners(null, new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE),
new EpochUpdateHolder());
}
@ -711,6 +711,7 @@ public class CommandsForKeySerializerTest
@Override public TopologyManager topology() { return null; }
@Override public long currentStamp() { return 0; }
@Override public void updateStamp() { throw new UnsupportedOperationException(); }
@Override public boolean isReplaying() { return false; }
@Override public long now() { return 0; }
@Override public long elapsed(TimeUnit unit) { return 0; }
}; }

6
test/unit/org/apache/cassandra/tcm/sequences/DropAccordTableTest.java
View File

@ -37,6 +37,8 @@ import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
@ -62,7 +64,9 @@ public class DropAccordTableTest
{
static
{
DatabaseDescriptor.clientInitialization();
DatabaseDescriptor.daemonInitialization();
// TODO (required): should this be a no-op service? Probably not! But this is what it was before (only relying on !accord.enabled check in AccordService.instance()...)
AccordService.unsafeSetNewAccordService(new IAccordService.NoOpAccordService());
}
private static final TransactionalMode[] ACCORD_ENABLED_MODES = Stream.of(TransactionalMode.values())

14
test/unit/org/apache/cassandra/utils/AccordGenerators.java
View File

@ -740,13 +740,11 @@ public class AccordGenerators
Gen<TinyEnumSet<Shard.Flag>> shardFlagsGen = shardFlagsGen();
return rs -> {
SortedArrayList<Node.Id> nodes = nodesGen.next(rs);
int maxFailures = Shard.maxToleratedFailures(nodes.size());
int slowQuorumSize = Shard.slowQuorumSize(nodes.size());
Set<Node.Id> fastPathElectorate = new TreeSet<>(select(nodes, nodes.size() == slowQuorumSize ? slowQuorumSize : rs.nextInt(slowQuorumSize, nodes.size())).next(rs));
List<Node.Id> nonFastPath = new ArrayList<>(Sets.difference(new HashSet<>(nodes), fastPathElectorate));
nonFastPath.sort(Comparator.naturalOrder());
Set<Node.Id> joining = new TreeSet<>(select(nonFastPath, nonFastPath.size() == 0 ? 0 : rs.nextInt(0, nonFastPath.size())).next(rs));
return Shard.create(range, nodes, fastPathElectorate, joining, shardFlagsGen.next(rs));
return Shard.create(range, nodes, fastPathElectorate, shardFlagsGen.next(rs));
};
}
@ -770,13 +768,19 @@ public class AccordGenerators
return rs -> {
long epoch = epochGen.nextLong(rs);
Ranges ranges = rangesGen.next(rs);
if (ranges.isEmpty()) return new Topology(epoch, new Shard[0]);
if (ranges.isEmpty())
return new Topology(epoch, new Shard[0]);
List<Shard> shards = new ArrayList<>(ranges.size());
for (Range range : ranges)
shards.add(shardGen(range).next(rs));
//TODO (coverage): staleNodes
return new Topology(epoch, shards.toArray(Shard[]::new));
Topology topology = new Topology(epoch, shards.toArray(Shard[]::new));
SortedArrayList<Node.Id> nodes = topology.nodes();
int hardRemovedCount = Math.min(rs.nextBoolean() ? 0 : rs.nextInt(0, 3), nodes.size());
SortedArrayList<Node.Id> hardRemoved = SortedArrayList.copyUnsorted(select(nodes, hardRemovedCount).next(rs), Node.Id[]::new);
return topology.withHardRemoved(hardRemoved);
};
}

3
test/unit/org/apache/cassandra/utils/CassandraGenerators.java
View File

@ -51,6 +51,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
@ -1872,7 +1873,7 @@ public final class CassandraGenerators
{
Gen<Set<Node.Id>> staleIdsGen = Generators.set(accordNodeId(), SourceDSL.integers().between(0, 10));
Gen<Epoch> epochGen = epochs();
return rnd -> new AccordStaleReplicas(staleIdsGen.generate(rnd), epochGen.generate(rnd));
return rnd -> new AccordStaleReplicas(SortedArrayList.copyUnsorted(staleIdsGen.generate(rnd), Node.Id[]::new), SortedArrayList.ofSorted(), epochGen.generate(rnd));
}
public static Gen<AccordFastPath> accordFastPath()

2
test/unit/org/apache/cassandra/utils/LargeBitSetSerializerTest.java
View File

@ -32,7 +32,7 @@ public class LargeBitSetSerializerTest
public void test()
{
@SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) DataOutputBuffer output = new DataOutputBuffer();
qt().forAll(largeBitSetGen()).check(bits -> Serializers.testSerde(output, LargeBitSetSerializer.instance, bits));
qt().forAll(largeBitSetGen()).check(bits -> Serializers.testSerde(output, SimpleBitSetSerializers.large, bits));
}
private static Gen<LargeBitSet> largeBitSetGen()